2024-11-26 23:24:51,399 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-26 23:24:51,411 main DEBUG Took 0.010044 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-26 23:24:51,411 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-26 23:24:51,411 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-26 23:24:51,412 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-26 23:24:51,413 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,419 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-26 23:24:51,431 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,432 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,433 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,433 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,434 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,435 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,435 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,435 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,435 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,436 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,436 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,437 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,437 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,438 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,438 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,438 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,438 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,439 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,439 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,440 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,440 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,441 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-26 23:24:51,441 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,441 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-26 23:24:51,442 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-26 23:24:51,444 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-26 23:24:51,445 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-26 23:24:51,446 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-26 23:24:51,447 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-26 23:24:51,447 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-26 23:24:51,455 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-26 23:24:51,457 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-26 23:24:51,458 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-26 23:24:51,459 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-26 23:24:51,459 main DEBUG createAppenders(={Console}) 2024-11-26 23:24:51,460 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-11-26 23:24:51,460 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-26 23:24:51,460 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-11-26 23:24:51,461 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-26 23:24:51,461 main DEBUG OutputStream closed 2024-11-26 23:24:51,461 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-26 23:24:51,461 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-26 23:24:51,462 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-11-26 23:24:51,527 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-26 23:24:51,529 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-26 23:24:51,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-26 23:24:51,530 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-26 23:24:51,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-26 23:24:51,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-26 23:24:51,531 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-26 23:24:51,532 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-26 23:24:51,532 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-26 23:24:51,532 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-26 23:24:51,533 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-26 23:24:51,533 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-26 23:24:51,533 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-26 23:24:51,533 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-26 23:24:51,534 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-26 23:24:51,534 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-26 23:24:51,534 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-26 23:24:51,535 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-26 23:24:51,537 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26 23:24:51,537 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6dab9b6d) with optional ClassLoader: null 2024-11-26 23:24:51,537 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-26 23:24:51,538 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6dab9b6d] started OK. 2024-11-26T23:24:51,759 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c 2024-11-26 23:24:51,761 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-26 23:24:51,762 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-26T23:24:51,771 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestLogRolling timeout: 13 mins 2024-11-26T23:24:51,804 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=304, ProcessCount=11, AvailableMemoryMB=7382 2024-11-26T23:24:51,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T23:24:51,821 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db, deleteOnExit=true 2024-11-26T23:24:51,821 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T23:24:51,822 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/test.cache.data in system properties and HBase conf 2024-11-26T23:24:51,823 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T23:24:51,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/hadoop.log.dir in system properties and HBase conf 2024-11-26T23:24:51,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T23:24:51,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T23:24:51,825 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T23:24:51,904 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-26T23:24:51,991 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T23:24:51,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:24:51,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:24:51,995 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T23:24:51,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:24:51,996 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T23:24:51,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T23:24:51,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:24:51,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:24:51,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T23:24:51,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/nfs.dump.dir in system properties and HBase conf 2024-11-26T23:24:51,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/java.io.tmpdir in system properties and HBase conf 2024-11-26T23:24:51,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:24:52,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T23:24:52,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T23:24:52,467 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:24:52,791 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-26T23:24:52,865 INFO [Time-limited test {}] log.Log(170): Logging initialized @2209ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-26T23:24:52,937 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:24:52,997 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:24:53,016 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:24:53,016 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:24:53,017 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:24:53,029 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:24:53,032 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:24:53,033 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:24:53,225 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@735fa16a{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/java.io.tmpdir/jetty-localhost-38037-hadoop-hdfs-3_4_1-tests_jar-_-any-1075943887840294651/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:24:53,232 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:38037} 2024-11-26T23:24:53,232 INFO [Time-limited test {}] server.Server(415): Started @2577ms 2024-11-26T23:24:53,259 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:24:53,611 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:24:53,617 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:24:53,618 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:24:53,618 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:24:53,619 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:24:53,620 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:24:53,620 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:24:53,740 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7b07d1ba{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/java.io.tmpdir/jetty-localhost-45515-hadoop-hdfs-3_4_1-tests_jar-_-any-4878947365085216377/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:24:53,740 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:45515} 2024-11-26T23:24:53,741 INFO [Time-limited test {}] server.Server(415): Started @3086ms 2024-11-26T23:24:53,795 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:24:53,915 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:24:53,922 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:24:53,923 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:24:53,923 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:24:53,924 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:24:53,925 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:24:53,925 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:24:54,053 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bf97579{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/java.io.tmpdir/jetty-localhost-44949-hadoop-hdfs-3_4_1-tests_jar-_-any-4008675598298728328/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:24:54,054 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:44949} 2024-11-26T23:24:54,054 INFO [Time-limited test {}] server.Server(415): Started @3399ms 2024-11-26T23:24:54,057 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:24:54,235 WARN [Thread-98 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db/data/data2/current/BP-2092199251-172.17.0.2-1732663492556/current, will proceed with Du for space computation calculation, 2024-11-26T23:24:54,235 WARN [Thread-97 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db/data/data1/current/BP-2092199251-172.17.0.2-1732663492556/current, will proceed with Du for space computation calculation, 2024-11-26T23:24:54,235 WARN [Thread-96 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db/data/data4/current/BP-2092199251-172.17.0.2-1732663492556/current, will proceed with Du for space computation calculation, 2024-11-26T23:24:54,235 WARN [Thread-95 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db/data/data3/current/BP-2092199251-172.17.0.2-1732663492556/current, will proceed with Du for space computation calculation, 2024-11-26T23:24:54,315 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:24:54,317 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:24:54,401 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x17bd416e40e8492c with lease ID 0x4244bc7a3a775b42: Processing first storage report for DS-3f016983-3a5b-47e1-afe0-da7448bd0666 from datanode DatanodeRegistration(127.0.0.1:36577, datanodeUuid=ff3656cd-c96f-4a2f-a2bd-e7e16af0dd78, infoPort=45917, infoSecurePort=0, ipcPort=44379, storageInfo=lv=-57;cid=testClusterID;nsid=1118747878;c=1732663492556) 2024-11-26T23:24:54,403 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17bd416e40e8492c with lease ID 0x4244bc7a3a775b42: from storage DS-3f016983-3a5b-47e1-afe0-da7448bd0666 node DatanodeRegistration(127.0.0.1:36577, datanodeUuid=ff3656cd-c96f-4a2f-a2bd-e7e16af0dd78, infoPort=45917, infoSecurePort=0, ipcPort=44379, storageInfo=lv=-57;cid=testClusterID;nsid=1118747878;c=1732663492556), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-26T23:24:54,403 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x54f7b2f149c5cef8 with lease ID 0x4244bc7a3a775b43: Processing first storage report for DS-7f05afdb-28a8-4705-94d9-fe8008406118 from datanode DatanodeRegistration(127.0.0.1:41255, datanodeUuid=533e0587-e020-45bd-b4e4-dfd9c6fc0426, infoPort=37109, infoSecurePort=0, ipcPort=36489, storageInfo=lv=-57;cid=testClusterID;nsid=1118747878;c=1732663492556) 2024-11-26T23:24:54,404 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x54f7b2f149c5cef8 with lease ID 0x4244bc7a3a775b43: from storage DS-7f05afdb-28a8-4705-94d9-fe8008406118 node DatanodeRegistration(127.0.0.1:41255, datanodeUuid=533e0587-e020-45bd-b4e4-dfd9c6fc0426, infoPort=37109, infoSecurePort=0, ipcPort=36489, storageInfo=lv=-57;cid=testClusterID;nsid=1118747878;c=1732663492556), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T23:24:54,404 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x17bd416e40e8492c with lease ID 0x4244bc7a3a775b42: Processing first storage report for DS-575feb27-520b-4193-9561-1ba05259622c from datanode DatanodeRegistration(127.0.0.1:36577, datanodeUuid=ff3656cd-c96f-4a2f-a2bd-e7e16af0dd78, infoPort=45917, infoSecurePort=0, ipcPort=44379, storageInfo=lv=-57;cid=testClusterID;nsid=1118747878;c=1732663492556) 2024-11-26T23:24:54,404 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x17bd416e40e8492c with lease ID 0x4244bc7a3a775b42: from storage DS-575feb27-520b-4193-9561-1ba05259622c node DatanodeRegistration(127.0.0.1:36577, datanodeUuid=ff3656cd-c96f-4a2f-a2bd-e7e16af0dd78, infoPort=45917, infoSecurePort=0, ipcPort=44379, storageInfo=lv=-57;cid=testClusterID;nsid=1118747878;c=1732663492556), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:24:54,405 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x54f7b2f149c5cef8 with lease ID 0x4244bc7a3a775b43: Processing first storage report for DS-89a266ec-2f28-4fd0-a054-99fc221c05bd from datanode DatanodeRegistration(127.0.0.1:41255, datanodeUuid=533e0587-e020-45bd-b4e4-dfd9c6fc0426, infoPort=37109, infoSecurePort=0, ipcPort=36489, storageInfo=lv=-57;cid=testClusterID;nsid=1118747878;c=1732663492556) 2024-11-26T23:24:54,405 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x54f7b2f149c5cef8 with lease ID 0x4244bc7a3a775b43: from storage DS-89a266ec-2f28-4fd0-a054-99fc221c05bd node DatanodeRegistration(127.0.0.1:41255, datanodeUuid=533e0587-e020-45bd-b4e4-dfd9c6fc0426, infoPort=37109, infoSecurePort=0, ipcPort=36489, storageInfo=lv=-57;cid=testClusterID;nsid=1118747878;c=1732663492556), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:24:54,432 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c 2024-11-26T23:24:54,507 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db/zookeeper_0, clientPort=59934, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T23:24:54,519 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59934 2024-11-26T23:24:54,529 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:24:54,532 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:24:54,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:24:54,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:24:55,176 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d with version=8 2024-11-26T23:24:55,177 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/hbase-staging 2024-11-26T23:24:55,268 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-26T23:24:55,522 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:24:55,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:24:55,533 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:24:55,538 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:24:55,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:24:55,538 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:24:55,677 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T23:24:55,738 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-26T23:24:55,747 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-26T23:24:55,750 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:24:55,780 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 16433 (auto-detected) 2024-11-26T23:24:55,781 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-26T23:24:55,805 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34413 2024-11-26T23:24:55,826 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34413 connecting to ZooKeeper ensemble=127.0.0.1:59934 2024-11-26T23:24:55,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:344130x0, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:24:55,864 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34413-0x1015698b1e90000 connected 2024-11-26T23:24:55,896 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:24:55,899 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:24:55,912 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:24:55,918 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d, hbase.cluster.distributed=false 2024-11-26T23:24:55,950 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:24:55,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34413 2024-11-26T23:24:55,957 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34413 2024-11-26T23:24:55,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34413 2024-11-26T23:24:55,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34413 2024-11-26T23:24:55,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34413 2024-11-26T23:24:56,087 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:24:56,089 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:24:56,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:24:56,090 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:24:56,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:24:56,090 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:24:56,093 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:24:56,095 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:24:56,096 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37045 2024-11-26T23:24:56,098 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37045 connecting to ZooKeeper ensemble=127.0.0.1:59934 2024-11-26T23:24:56,099 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:24:56,103 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:24:56,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370450x0, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:24:56,111 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:370450x0, quorum=127.0.0.1:59934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:24:56,112 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37045-0x1015698b1e90001 connected 2024-11-26T23:24:56,116 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:24:56,124 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:24:56,126 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T23:24:56,131 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:24:56,132 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37045 2024-11-26T23:24:56,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37045 2024-11-26T23:24:56,133 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37045 2024-11-26T23:24:56,134 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37045 2024-11-26T23:24:56,134 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37045 2024-11-26T23:24:56,151 DEBUG [M:0;c707e9ace675:34413 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c707e9ace675:34413 2024-11-26T23:24:56,152 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c707e9ace675,34413,1732663495322 2024-11-26T23:24:56,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:24:56,159 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:24:56,161 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c707e9ace675,34413,1732663495322 2024-11-26T23:24:56,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T23:24:56,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:56,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:56,183 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T23:24:56,184 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c707e9ace675,34413,1732663495322 from backup master directory 2024-11-26T23:24:56,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c707e9ace675,34413,1732663495322 2024-11-26T23:24:56,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:24:56,187 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:24:56,188 WARN [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:24:56,188 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c707e9ace675,34413,1732663495322 2024-11-26T23:24:56,190 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-26T23:24:56,191 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-26T23:24:56,252 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/hbase.id] with ID: b465dc02-47b2-42ac-82c1-eefab02101bb 2024-11-26T23:24:56,252 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/.tmp/hbase.id 2024-11-26T23:24:56,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:24:56,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:24:56,265 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/.tmp/hbase.id]:[hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/hbase.id] 2024-11-26T23:24:56,308 INFO [master/c707e9ace675:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:24:56,313 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T23:24:56,332 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-26T23:24:56,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:56,335 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:56,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:24:56,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:24:56,368 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:24:56,371 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T23:24:56,377 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:24:56,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:24:56,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:24:56,433 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store 2024-11-26T23:24:56,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:24:56,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:24:56,458 INFO [master/c707e9ace675:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-26T23:24:56,462 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:24:56,464 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:24:56,464 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:24:56,464 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:24:56,466 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:24:56,466 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:24:56,467 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:24:56,468 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663496464Disabling compacts and flushes for region at 1732663496464Disabling writes for close at 1732663496466 (+2 ms)Writing region close event to WAL at 1732663496467 (+1 ms)Closed at 1732663496467 2024-11-26T23:24:56,470 WARN [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/.initializing 2024-11-26T23:24:56,471 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/WALs/c707e9ace675,34413,1732663495322 2024-11-26T23:24:56,494 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C34413%2C1732663495322, suffix=, logDir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/WALs/c707e9ace675,34413,1732663495322, archiveDir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/oldWALs, maxLogs=10 2024-11-26T23:24:56,505 INFO [master/c707e9ace675:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C34413%2C1732663495322.1732663496500 2024-11-26T23:24:56,529 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/WALs/c707e9ace675,34413,1732663495322/c707e9ace675%2C34413%2C1732663495322.1732663496500 2024-11-26T23:24:56,537 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37109:37109),(127.0.0.1/127.0.0.1:45917:45917)] 2024-11-26T23:24:56,538 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:24:56,539 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:24:56,542 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,543 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,580 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,607 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T23:24:56,611 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:56,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:24:56,614 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,617 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T23:24:56,617 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:56,618 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:24:56,619 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,621 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T23:24:56,621 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:56,622 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:24:56,623 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,625 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T23:24:56,625 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:56,626 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:24:56,626 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,630 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,631 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,636 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,636 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,640 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:24:56,643 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:24:56,647 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:24:56,648 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=868885, jitterRate=0.10484451055526733}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:24:56,654 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732663496555Initializing all the Stores at 1732663496557 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663496558 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663496558Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663496559 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663496559Cleaning up temporary data from old regions at 1732663496636 (+77 ms)Region opened successfully at 1732663496654 (+18 ms) 2024-11-26T23:24:56,655 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T23:24:56,692 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c4ce236, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:24:56,726 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T23:24:56,738 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T23:24:56,738 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T23:24:56,741 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T23:24:56,742 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-26T23:24:56,747 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-26T23:24:56,747 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T23:24:56,772 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T23:24:56,780 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T23:24:56,782 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T23:24:56,785 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T23:24:56,786 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T23:24:56,788 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T23:24:56,790 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T23:24:56,793 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T23:24:56,795 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T23:24:56,796 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T23:24:56,797 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T23:24:56,814 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T23:24:56,815 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T23:24:56,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:24:56,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:24:56,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:56,820 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:56,823 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c707e9ace675,34413,1732663495322, sessionid=0x1015698b1e90000, setting cluster-up flag (Was=false) 2024-11-26T23:24:56,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:56,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:56,844 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T23:24:56,846 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,34413,1732663495322 2024-11-26T23:24:56,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:56,851 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:56,857 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T23:24:56,859 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,34413,1732663495322 2024-11-26T23:24:56,865 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T23:24:56,937 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T23:24:56,939 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(746): ClusterId : b465dc02-47b2-42ac-82c1-eefab02101bb 2024-11-26T23:24:56,941 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:24:56,946 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:24:56,947 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:24:56,947 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T23:24:56,950 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:24:56,951 DEBUG [RS:0;c707e9ace675:37045 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e676e5b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:24:56,955 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T23:24:56,964 DEBUG [RS:0;c707e9ace675:37045 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c707e9ace675:37045 2024-11-26T23:24:56,963 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c707e9ace675,34413,1732663495322 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T23:24:56,967 INFO [RS:0;c707e9ace675:37045 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:24:56,967 INFO [RS:0;c707e9ace675:37045 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:24:56,967 DEBUG [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:24:56,970 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,34413,1732663495322 with port=37045, startcode=1732663496045 2024-11-26T23:24:56,972 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:24:56,972 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:24:56,972 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:24:56,972 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:24:56,973 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c707e9ace675:0, corePoolSize=10, maxPoolSize=10 2024-11-26T23:24:56,973 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:56,973 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:24:56,973 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:56,978 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:24:56,979 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732663526978 2024-11-26T23:24:56,979 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T23:24:56,980 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T23:24:56,981 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T23:24:56,983 DEBUG [RS:0;c707e9ace675:37045 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:24:56,985 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:56,985 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T23:24:56,985 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T23:24:56,986 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T23:24:56,986 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T23:24:56,986 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T23:24:56,987 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:56,993 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T23:24:56,994 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T23:24:56,995 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T23:24:56,997 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T23:24:56,997 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T23:24:56,999 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663496999,5,FailOnTimeoutGroup] 2024-11-26T23:24:57,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:24:57,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:24:57,001 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663497000,5,FailOnTimeoutGroup] 2024-11-26T23:24:57,001 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,001 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T23:24:57,003 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,003 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,003 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T23:24:57,004 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d 2024-11-26T23:24:57,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:24:57,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:24:57,023 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:24:57,039 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:24:57,044 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:24:57,044 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:57,045 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:24:57,046 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:24:57,051 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:24:57,052 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:57,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:24:57,053 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:24:57,057 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:24:57,057 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:57,058 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:24:57,058 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:24:57,061 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:24:57,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:57,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:24:57,063 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:24:57,064 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740 2024-11-26T23:24:57,065 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740 2024-11-26T23:24:57,066 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56035, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:24:57,069 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:24:57,069 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:24:57,070 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:24:57,073 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:24:57,074 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34413 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,37045,1732663496045 2024-11-26T23:24:57,077 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34413 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,37045,1732663496045 2024-11-26T23:24:57,081 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:24:57,082 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=747027, jitterRate=-0.050107285380363464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:24:57,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732663497023Initializing all the Stores at 1732663497025 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663497025Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663497039 (+14 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663497039Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663497039Cleaning up temporary data from old regions at 1732663497069 (+30 ms)Region opened successfully at 1732663497085 (+16 ms) 2024-11-26T23:24:57,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:24:57,085 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:24:57,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:24:57,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:24:57,085 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:24:57,087 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:24:57,087 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663497085Disabling compacts and flushes for region at 1732663497085Disabling writes for close at 1732663497085Writing region close event to WAL at 1732663497086 (+1 ms)Closed at 1732663497087 (+1 ms) 2024-11-26T23:24:57,090 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:24:57,091 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T23:24:57,094 DEBUG [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d 2024-11-26T23:24:57,094 DEBUG [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34219 2024-11-26T23:24:57,094 DEBUG [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:24:57,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T23:24:57,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:24:57,100 DEBUG [RS:0;c707e9ace675:37045 {}] zookeeper.ZKUtil(111): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,37045,1732663496045 2024-11-26T23:24:57,100 WARN [RS:0;c707e9ace675:37045 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:24:57,100 INFO [RS:0;c707e9ace675:37045 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:24:57,101 DEBUG [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045 2024-11-26T23:24:57,102 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,37045,1732663496045] 2024-11-26T23:24:57,108 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:24:57,112 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T23:24:57,131 INFO [RS:0;c707e9ace675:37045 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:24:57,146 INFO [RS:0;c707e9ace675:37045 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:24:57,151 INFO [RS:0;c707e9ace675:37045 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:24:57,151 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,152 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:24:57,158 INFO [RS:0;c707e9ace675:37045 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:24:57,160 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,160 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:57,160 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:57,161 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:57,161 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:57,161 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:57,161 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:24:57,161 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:57,162 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:57,162 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:57,162 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:57,162 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:57,163 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:24:57,163 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:24:57,163 DEBUG [RS:0;c707e9ace675:37045 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:24:57,164 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,165 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,165 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,165 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,165 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,165 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,37045,1732663496045-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:24:57,193 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:24:57,195 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,37045,1732663496045-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,196 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,196 INFO [RS:0;c707e9ace675:37045 {}] regionserver.Replication(171): c707e9ace675,37045,1732663496045 started 2024-11-26T23:24:57,223 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,223 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,37045,1732663496045, RpcServer on c707e9ace675/172.17.0.2:37045, sessionid=0x1015698b1e90001 2024-11-26T23:24:57,224 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:24:57,225 DEBUG [RS:0;c707e9ace675:37045 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,37045,1732663496045 2024-11-26T23:24:57,225 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,37045,1732663496045' 2024-11-26T23:24:57,225 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:24:57,226 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:24:57,227 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:24:57,227 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:24:57,228 DEBUG [RS:0;c707e9ace675:37045 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,37045,1732663496045 2024-11-26T23:24:57,228 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,37045,1732663496045' 2024-11-26T23:24:57,228 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:24:57,229 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:24:57,229 DEBUG [RS:0;c707e9ace675:37045 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:24:57,230 INFO [RS:0;c707e9ace675:37045 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:24:57,230 INFO [RS:0;c707e9ace675:37045 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:24:57,263 WARN [c707e9ace675:34413 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T23:24:57,339 INFO [RS:0;c707e9ace675:37045 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C37045%2C1732663496045, suffix=, logDir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045, archiveDir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/oldWALs, maxLogs=32 2024-11-26T23:24:57,341 INFO [RS:0;c707e9ace675:37045 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C37045%2C1732663496045.1732663497341 2024-11-26T23:24:57,350 INFO [RS:0;c707e9ace675:37045 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663497341 2024-11-26T23:24:57,351 DEBUG [RS:0;c707e9ace675:37045 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37109:37109),(127.0.0.1/127.0.0.1:45917:45917)] 2024-11-26T23:24:57,515 DEBUG [c707e9ace675:34413 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T23:24:57,527 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c707e9ace675,37045,1732663496045 2024-11-26T23:24:57,534 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,37045,1732663496045, state=OPENING 2024-11-26T23:24:57,539 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T23:24:57,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:57,541 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:24:57,541 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:24:57,541 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:24:57,543 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:24:57,546 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,37045,1732663496045}] 2024-11-26T23:24:57,721 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:24:57,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47719, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:24:57,736 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T23:24:57,737 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:24:57,740 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C37045%2C1732663496045.meta, suffix=.meta, logDir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045, archiveDir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/oldWALs, maxLogs=32 2024-11-26T23:24:57,742 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C37045%2C1732663496045.meta.1732663497742.meta 2024-11-26T23:24:57,750 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.meta.1732663497742.meta 2024-11-26T23:24:57,751 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45917:45917),(127.0.0.1/127.0.0.1:37109:37109)] 2024-11-26T23:24:57,752 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:24:57,754 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T23:24:57,756 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T23:24:57,762 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T23:24:57,768 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T23:24:57,768 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:24:57,769 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T23:24:57,769 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T23:24:57,772 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:24:57,773 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:24:57,774 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:57,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:24:57,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:24:57,776 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:24:57,776 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:57,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:24:57,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:24:57,778 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:24:57,778 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:57,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:24:57,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:24:57,781 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:24:57,781 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:57,782 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:24:57,782 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:24:57,783 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740 2024-11-26T23:24:57,786 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740 2024-11-26T23:24:57,788 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:24:57,788 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:24:57,789 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:24:57,791 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:24:57,793 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=754401, jitterRate=-0.040729716420173645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:24:57,794 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T23:24:57,796 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732663497770Writing region info on filesystem at 1732663497770Initializing all the Stores at 1732663497771 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663497772 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663497772Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663497772Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663497772Cleaning up temporary data from old regions at 1732663497788 (+16 ms)Running coprocessor post-open hooks at 1732663497794 (+6 ms)Region opened successfully at 1732663497795 (+1 ms) 2024-11-26T23:24:57,802 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732663497712 2024-11-26T23:24:57,814 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T23:24:57,814 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T23:24:57,815 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,37045,1732663496045 2024-11-26T23:24:57,817 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,37045,1732663496045, state=OPEN 2024-11-26T23:24:57,824 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:24:57,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:24:57,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:24:57,825 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:24:57,825 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c707e9ace675,37045,1732663496045 2024-11-26T23:24:57,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T23:24:57,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,37045,1732663496045 in 280 msec 2024-11-26T23:24:57,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T23:24:57,839 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 735 msec 2024-11-26T23:24:57,840 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:24:57,841 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T23:24:57,861 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:24:57,862 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,37045,1732663496045, seqNum=-1] 2024-11-26T23:24:57,883 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:24:57,885 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44881, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:24:57,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0180 sec 2024-11-26T23:24:57,913 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732663497913, completionTime=-1 2024-11-26T23:24:57,916 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T23:24:57,916 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T23:24:57,943 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T23:24:57,943 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732663557943 2024-11-26T23:24:57,943 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732663617943 2024-11-26T23:24:57,943 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 26 msec 2024-11-26T23:24:57,946 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34413,1732663495322-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,946 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34413,1732663495322-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,946 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34413,1732663495322-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,948 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c707e9ace675:34413, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,948 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,948 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:57,954 DEBUG [master/c707e9ace675:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T23:24:57,974 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.785sec 2024-11-26T23:24:57,975 INFO [master/c707e9ace675:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T23:24:57,976 INFO [master/c707e9ace675:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T23:24:57,977 INFO [master/c707e9ace675:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T23:24:57,977 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T23:24:57,978 INFO [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T23:24:57,978 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34413,1732663495322-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:24:57,979 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34413,1732663495322-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T23:24:57,987 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T23:24:57,987 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T23:24:57,988 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34413,1732663495322-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:24:58,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4731d90b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:24:58,052 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-26T23:24:58,052 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-26T23:24:58,055 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,34413,-1 for getting cluster id 2024-11-26T23:24:58,058 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:24:58,070 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b465dc02-47b2-42ac-82c1-eefab02101bb' 2024-11-26T23:24:58,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:24:58,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b465dc02-47b2-42ac-82c1-eefab02101bb" 2024-11-26T23:24:58,073 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d9aa4cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:24:58,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,34413,-1] 2024-11-26T23:24:58,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:24:58,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:24:58,080 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:24:58,083 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb3700d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:24:58,083 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:24:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,37045,1732663496045, seqNum=-1] 2024-11-26T23:24:58,091 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:24:58,094 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52642, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:24:58,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c707e9ace675,34413,1732663495322 2024-11-26T23:24:58,115 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:24:58,122 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-26T23:24:58,126 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:24:58,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,34413,1732663495322 2024-11-26T23:24:58,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@794fd916 2024-11-26T23:24:58,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:24:58,138 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:24:58,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34413 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-26T23:24:58,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34413 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-26T23:24:58,145 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34413 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testSlowSyncLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:24:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34413 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling 2024-11-26T23:24:58,156 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:24:58,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34413 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testSlowSyncLogRolling" procId is: 4 2024-11-26T23:24:58,158 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:58,160 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:24:58,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:24:58,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741835_1011 (size=389) 2024-11-26T23:24:58,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741835_1011 (size=389) 2024-11-26T23:24:58,208 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fb3bce74abd382c981bf9c89508ce7a0, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testSlowSyncLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d 2024-11-26T23:24:58,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741836_1012 (size=72) 2024-11-26T23:24:58,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741836_1012 (size=72) 2024-11-26T23:24:58,218 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:24:58,218 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing fb3bce74abd382c981bf9c89508ce7a0, disabling compactions & flushes 2024-11-26T23:24:58,219 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:24:58,219 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:24:58,219 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. after waiting 0 ms 2024-11-26T23:24:58,219 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:24:58,219 INFO [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:24:58,219 DEBUG [RegionOpenAndInit-TestLogRolling-testSlowSyncLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for fb3bce74abd382c981bf9c89508ce7a0: Waiting for close lock at 1732663498218Disabling compacts and flushes for region at 1732663498218Disabling writes for close at 1732663498219 (+1 ms)Writing region close event to WAL at 1732663498219Closed at 1732663498219 2024-11-26T23:24:58,221 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:24:58,226 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.","families":{"info":[{"qualifier":"regioninfo","vlen":71,"tag":[],"timestamp":"1732663498221"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732663498221"}]},"ts":"1732663498221"} 2024-11-26T23:24:58,231 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:24:58,234 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:24:58,237 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663498234"}]},"ts":"1732663498234"} 2024-11-26T23:24:58,241 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLING in hbase:meta 2024-11-26T23:24:58,243 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=fb3bce74abd382c981bf9c89508ce7a0, ASSIGN}] 2024-11-26T23:24:58,246 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=fb3bce74abd382c981bf9c89508ce7a0, ASSIGN 2024-11-26T23:24:58,248 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=fb3bce74abd382c981bf9c89508ce7a0, ASSIGN; state=OFFLINE, location=c707e9ace675,37045,1732663496045; forceNewPlan=false, retain=false 2024-11-26T23:24:58,399 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fb3bce74abd382c981bf9c89508ce7a0, regionState=OPENING, regionLocation=c707e9ace675,37045,1732663496045 2024-11-26T23:24:58,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=fb3bce74abd382c981bf9c89508ce7a0, ASSIGN because future has completed 2024-11-26T23:24:58,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb3bce74abd382c981bf9c89508ce7a0, server=c707e9ace675,37045,1732663496045}] 2024-11-26T23:24:58,565 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:24:58,565 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fb3bce74abd382c981bf9c89508ce7a0, NAME => 'TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:24:58,566 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testSlowSyncLogRolling fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:24:58,566 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:24:58,566 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:24:58,566 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:24:58,568 INFO [StoreOpener-fb3bce74abd382c981bf9c89508ce7a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:24:58,571 INFO [StoreOpener-fb3bce74abd382c981bf9c89508ce7a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb3bce74abd382c981bf9c89508ce7a0 columnFamilyName info 2024-11-26T23:24:58,571 DEBUG [StoreOpener-fb3bce74abd382c981bf9c89508ce7a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:24:58,572 INFO [StoreOpener-fb3bce74abd382c981bf9c89508ce7a0-1 {}] regionserver.HStore(327): Store=fb3bce74abd382c981bf9c89508ce7a0/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:24:58,572 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:24:58,574 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:24:58,574 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:24:58,575 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:24:58,575 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:24:58,577 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:24:58,580 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:24:58,581 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fb3bce74abd382c981bf9c89508ce7a0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=831139, jitterRate=0.056848421692848206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:24:58,581 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:24:58,582 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fb3bce74abd382c981bf9c89508ce7a0: Running coprocessor pre-open hook at 1732663498566Writing region info on filesystem at 1732663498566Initializing all the Stores at 1732663498568 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663498568Cleaning up temporary data from old regions at 1732663498575 (+7 ms)Running coprocessor post-open hooks at 1732663498581 (+6 ms)Region opened successfully at 1732663498582 (+1 ms) 2024-11-26T23:24:58,584 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0., pid=6, masterSystemTime=1732663498558 2024-11-26T23:24:58,588 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:24:58,588 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:24:58,589 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fb3bce74abd382c981bf9c89508ce7a0, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,37045,1732663496045 2024-11-26T23:24:58,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb3bce74abd382c981bf9c89508ce7a0, server=c707e9ace675,37045,1732663496045 because future has completed 2024-11-26T23:24:58,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T23:24:58,600 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fb3bce74abd382c981bf9c89508ce7a0, server=c707e9ace675,37045,1732663496045 in 191 msec 2024-11-26T23:24:58,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T23:24:58,604 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testSlowSyncLogRolling, region=fb3bce74abd382c981bf9c89508ce7a0, ASSIGN in 356 msec 2024-11-26T23:24:58,605 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:24:58,605 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testSlowSyncLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663498605"}]},"ts":"1732663498605"} 2024-11-26T23:24:58,608 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testSlowSyncLogRolling, state=ENABLED in hbase:meta 2024-11-26T23:24:58,610 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:24:58,613 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testSlowSyncLogRolling in 462 msec 2024-11-26T23:25:03,271 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-26T23:25:03,318 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T23:25:03,319 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testSlowSyncLogRolling' 2024-11-26T23:25:05,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T23:25:05,735 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:05,737 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-26T23:25:05,737 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:05,738 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:25:05,738 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T23:25:05,738 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T23:25:05,738 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-26T23:25:08,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34413 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:25:08,248 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testSlowSyncLogRolling completed 2024-11-26T23:25:08,251 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testSlowSyncLogRolling,, stopping at row=TestLogRolling-testSlowSyncLogRolling ,, for max=2147483647 with caching=100 2024-11-26T23:25:08,257 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testSlowSyncLogRolling 2024-11-26T23:25:08,258 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:25:08,259 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C37045%2C1732663496045.1732663508258 2024-11-26T23:25:08,268 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:08,268 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:08,268 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:08,268 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:08,269 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:08,269 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663497341 with entries=1, filesize=443 B; new WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663508258 2024-11-26T23:25:08,270 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45917:45917),(127.0.0.1/127.0.0.1:37109:37109)] 2024-11-26T23:25:08,271 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663497341 is not closed yet, will try archiving it next time 2024-11-26T23:25:08,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741833_1009 (size=451) 2024-11-26T23:25:08,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741833_1009 (size=451) 2024-11-26T23:25:08,274 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663497341 to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/oldWALs/c707e9ace675%2C37045%2C1732663496045.1732663497341 2024-11-26T23:25:08,281 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testSlowSyncLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0., hostname=c707e9ace675,37045,1732663496045, seqNum=2] 2024-11-26T23:25:20,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37045 {}] regionserver.HRegion(8855): Flush requested on fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:25:20,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb3bce74abd382c981bf9c89508ce7a0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-26T23:25:20,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/07b58ef5960549cc85f858315ca0cbbb is 1080, key is row0001/info:/1732663508283/Put/seqid=0 2024-11-26T23:25:20,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741838_1014 (size=12509) 2024-11-26T23:25:20,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741838_1014 (size=12509) 2024-11-26T23:25:20,395 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/07b58ef5960549cc85f858315ca0cbbb 2024-11-26T23:25:20,441 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/07b58ef5960549cc85f858315ca0cbbb as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/07b58ef5960549cc85f858315ca0cbbb 2024-11-26T23:25:20,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/07b58ef5960549cc85f858315ca0cbbb, entries=7, sequenceid=11, filesize=12.2 K 2024-11-26T23:25:20,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for fb3bce74abd382c981bf9c89508ce7a0 in 140ms, sequenceid=11, compaction requested=false 2024-11-26T23:25:20,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb3bce74abd382c981bf9c89508ce7a0: 2024-11-26T23:25:24,429 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:25:28,327 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C37045%2C1732663496045.1732663528327 2024-11-26T23:25:28,536 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 206 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK], DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK]] 2024-11-26T23:25:28,536 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:28,536 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:28,536 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:28,536 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:28,537 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:28,537 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663508258 with entries=12, filesize=12.10 KB; new WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663528327 2024-11-26T23:25:28,538 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37109:37109),(127.0.0.1/127.0.0.1:45917:45917)] 2024-11-26T23:25:28,538 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663508258 is not closed yet, will try archiving it next time 2024-11-26T23:25:28,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741837_1013 (size=12399) 2024-11-26T23:25:28,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741837_1013 (size=12399) 2024-11-26T23:25:28,741 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:30,945 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:33,150 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:35,353 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:35,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37045 {}] regionserver.HRegion(8855): Flush requested on fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:25:35,354 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb3bce74abd382c981bf9c89508ce7a0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-26T23:25:35,556 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:35,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/744436871fba47948dc671164cb7417f is 1080, key is row0008/info:/1732663522316/Put/seqid=0 2024-11-26T23:25:35,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741840_1016 (size=12509) 2024-11-26T23:25:35,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741840_1016 (size=12509) 2024-11-26T23:25:35,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/744436871fba47948dc671164cb7417f 2024-11-26T23:25:35,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/744436871fba47948dc671164cb7417f as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/744436871fba47948dc671164cb7417f 2024-11-26T23:25:35,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/744436871fba47948dc671164cb7417f, entries=7, sequenceid=21, filesize=12.2 K 2024-11-26T23:25:35,792 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:35,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for fb3bce74abd382c981bf9c89508ce7a0 in 439ms, sequenceid=21, compaction requested=false 2024-11-26T23:25:35,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb3bce74abd382c981bf9c89508ce7a0: 2024-11-26T23:25:35,792 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=24.4 K, sizeToCheck=16.0 K 2024-11-26T23:25:35,792 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:25:35,793 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/07b58ef5960549cc85f858315ca0cbbb because midkey is the same as first or last row 2024-11-26T23:25:37,557 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:37,990 INFO [master/c707e9ace675:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T23:25:37,990 INFO [master/c707e9ace675:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T23:25:39,761 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:39,764 WARN [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(2201): Requesting log roll because we exceeded slow sync threshold; count=8, threshold=5, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:39,765 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C37045%2C1732663496045:(num 1732663528327) roll requested 2024-11-26T23:25:39,765 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C37045%2C1732663496045.1732663539765 2024-11-26T23:25:39,973 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 205 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:39,973 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:39,973 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:39,973 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:39,974 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:39,974 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:39,974 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663528327 with entries=8, filesize=7.55 KB; new WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663539765 2024-11-26T23:25:39,975 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37109:37109),(127.0.0.1/127.0.0.1:45917:45917)] 2024-11-26T23:25:39,975 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663528327 is not closed yet, will try archiving it next time 2024-11-26T23:25:39,975 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663508258 to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/oldWALs/c707e9ace675%2C37045%2C1732663496045.1732663508258 2024-11-26T23:25:39,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741839_1015 (size=7739) 2024-11-26T23:25:39,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741839_1015 (size=7739) 2024-11-26T23:25:41,965 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:43,566 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region fb3bce74abd382c981bf9c89508ce7a0, had cached 0 bytes from a total of 25018 2024-11-26T23:25:44,170 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:46,374 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:48,578 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 201 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:50,580 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T23:25:50,581 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C37045%2C1732663496045.1732663550581 2024-11-26T23:25:54,430 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:25:55,594 INFO [Time-limited test {}] wal.AbstractFSWAL(1368): Slow sync cost: 5010 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:55,596 WARN [Time-limited test {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5010 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:25:55,596 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C37045%2C1732663496045:(num 1732663550581) roll requested 2024-11-26T23:25:55,597 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:55,597 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:55,597 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:55,597 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:55,598 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:25:55,598 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663539765 with entries=4, filesize=4.63 KB; new WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663550581 2024-11-26T23:25:55,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741841_1017 (size=4753) 2024-11-26T23:25:55,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741841_1017 (size=4753) 2024-11-26T23:25:55,616 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37109:37109),(127.0.0.1/127.0.0.1:45917:45917)] 2024-11-26T23:25:55,617 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C37045%2C1732663496045.1732663555617 2024-11-26T23:26:00,621 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:26:00,622 WARN [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:26:00,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37045 {}] regionserver.HRegion(8855): Flush requested on fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:26:00,622 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb3bce74abd382c981bf9c89508ce7a0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-26T23:26:00,632 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5011 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:26:00,632 WARN [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5011 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:26:02,623 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T23:26:05,625 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:26:05,625 WARN [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK], DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK]] 2024-11-26T23:26:05,625 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:05,625 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:05,625 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:05,625 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:05,626 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:05,626 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663550581 with entries=2, filesize=1.52 KB; new WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663555617 2024-11-26T23:26:05,627 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45917:45917),(127.0.0.1/127.0.0.1:37109:37109)] 2024-11-26T23:26:05,627 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663550581 is not closed yet, will try archiving it next time 2024-11-26T23:26:05,627 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C37045%2C1732663496045:(num 1732663555617) roll requested 2024-11-26T23:26:05,627 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C37045%2C1732663496045.1732663565627 2024-11-26T23:26:05,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741842_1018 (size=1569) 2024-11-26T23:26:05,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741842_1018 (size=1569) 2024-11-26T23:26:05,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/86f4f67398e946638aa4d9be6b71aacd is 1080, key is row0015/info:/1732663537356/Put/seqid=0 2024-11-26T23:26:05,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741844_1020 (size=12509) 2024-11-26T23:26:05,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741844_1020 (size=12509) 2024-11-26T23:26:06,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=31 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/86f4f67398e946638aa4d9be6b71aacd 2024-11-26T23:26:06,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/86f4f67398e946638aa4d9be6b71aacd as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/86f4f67398e946638aa4d9be6b71aacd 2024-11-26T23:26:06,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/86f4f67398e946638aa4d9be6b71aacd, entries=7, sequenceid=31, filesize=12.2 K 2024-11-26T23:26:10,639 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1368): Slow sync cost: 5008 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK], DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK]] 2024-11-26T23:26:10,639 WARN [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5008 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK], DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK]] 2024-11-26T23:26:11,067 INFO [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1368): Slow sync cost: 5001 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK], DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK]] 2024-11-26T23:26:11,067 WARN [FSHLog-0-hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d-prefix:c707e9ace675,37045,1732663496045 {}] wal.AbstractFSWAL(1374): Requesting log roll because we exceeded slow sync threshold; time=5001 ms, threshold=5000 ms, current pipeline: [DatanodeInfoWithStorage[127.0.0.1:36577,DS-3f016983-3a5b-47e1-afe0-da7448bd0666,DISK], DatanodeInfoWithStorage[127.0.0.1:41255,DS-7f05afdb-28a8-4705-94d9-fe8008406118,DISK]] 2024-11-26T23:26:11,067 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for fb3bce74abd382c981bf9c89508ce7a0 in 10445ms, sequenceid=31, compaction requested=true 2024-11-26T23:26:11,067 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb3bce74abd382c981bf9c89508ce7a0: 2024-11-26T23:26:11,067 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,068 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,068 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=36.6 K, sizeToCheck=16.0 K 2024-11-26T23:26:11,068 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:11,068 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,068 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/07b58ef5960549cc85f858315ca0cbbb because midkey is the same as first or last row 2024-11-26T23:26:11,068 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,068 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663555617 with entries=1, filesize=430 B; new WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663565627 2024-11-26T23:26:11,069 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37109:37109),(127.0.0.1/127.0.0.1:45917:45917)] 2024-11-26T23:26:11,069 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663555617 is not closed yet, will try archiving it next time 2024-11-26T23:26:11,069 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663528327 to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/oldWALs/c707e9ace675%2C37045%2C1732663496045.1732663528327 2024-11-26T23:26:11,069 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C37045%2C1732663496045:(num 1732663571069) roll requested 2024-11-26T23:26:11,070 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C37045%2C1732663496045.1732663571069 2024-11-26T23:26:11,070 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store fb3bce74abd382c981bf9c89508ce7a0:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:26:11,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741843_1019 (size=438) 2024-11-26T23:26:11,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741843_1019 (size=438) 2024-11-26T23:26:11,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:26:11,073 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663539765 to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/oldWALs/c707e9ace675%2C37045%2C1732663496045.1732663539765 2024-11-26T23:26:11,075 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663550581 to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/oldWALs/c707e9ace675%2C37045%2C1732663496045.1732663550581 2024-11-26T23:26:11,075 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:26:11,077 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663555617 to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/oldWALs/c707e9ace675%2C37045%2C1732663496045.1732663555617 2024-11-26T23:26:11,078 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37527 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:26:11,080 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.HStore(1541): fb3bce74abd382c981bf9c89508ce7a0/info is initiating minor compaction (all files) 2024-11-26T23:26:11,081 INFO [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of fb3bce74abd382c981bf9c89508ce7a0/info in TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:26:11,081 INFO [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/07b58ef5960549cc85f858315ca0cbbb, hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/744436871fba47948dc671164cb7417f, hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/86f4f67398e946638aa4d9be6b71aacd] into tmpdir=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp, totalSize=36.6 K 2024-11-26T23:26:11,093 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07b58ef5960549cc85f858315ca0cbbb, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732663508283 2024-11-26T23:26:11,100 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744436871fba47948dc671164cb7417f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=21, earliestPutTs=1732663522316 2024-11-26T23:26:11,102 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86f4f67398e946638aa4d9be6b71aacd, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=31, earliestPutTs=1732663537356 2024-11-26T23:26:11,105 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,105 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,105 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,105 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,105 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,106 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663565627 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663571069 2024-11-26T23:26:11,108 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45917:45917),(127.0.0.1/127.0.0.1:37109:37109)] 2024-11-26T23:26:11,108 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663565627 is not closed yet, will try archiving it next time 2024-11-26T23:26:11,108 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C37045%2C1732663496045.1732663571108 2024-11-26T23:26:11,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741845_1021 (size=93) 2024-11-26T23:26:11,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741845_1021 (size=93) 2024-11-26T23:26:11,123 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,123 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,123 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,124 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,124 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:11,124 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663571069 with entries=1, filesize=1.22 KB; new WAL /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663571108 2024-11-26T23:26:11,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741846_1022 (size=1258) 2024-11-26T23:26:11,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741846_1022 (size=1258) 2024-11-26T23:26:11,132 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663565627 is not closed yet, will try archiving it next time 2024-11-26T23:26:11,133 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:45917:45917),(127.0.0.1/127.0.0.1:37109:37109)] 2024-11-26T23:26:11,133 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663565627 is not closed yet, will try archiving it next time 2024-11-26T23:26:11,160 INFO [RS:0;c707e9ace675:37045-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): fb3bce74abd382c981bf9c89508ce7a0#info#compaction#3 average throughput is 10.77 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:26:11,161 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/d35daee0dc594d3596f2e837ca095069 is 1080, key is row0001/info:/1732663508283/Put/seqid=0 2024-11-26T23:26:11,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741848_1024 (size=27710) 2024-11-26T23:26:11,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741848_1024 (size=27710) 2024-11-26T23:26:11,203 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/d35daee0dc594d3596f2e837ca095069 as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/d35daee0dc594d3596f2e837ca095069 2024-11-26T23:26:11,221 INFO [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in fb3bce74abd382c981bf9c89508ce7a0/info of fb3bce74abd382c981bf9c89508ce7a0 into d35daee0dc594d3596f2e837ca095069(size=27.1 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:26:11,221 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for fb3bce74abd382c981bf9c89508ce7a0: 2024-11-26T23:26:11,224 INFO [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0., storeName=fb3bce74abd382c981bf9c89508ce7a0/info, priority=13, startTime=1732663571069; duration=0sec 2024-11-26T23:26:11,225 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-26T23:26:11,225 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:11,225 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/d35daee0dc594d3596f2e837ca095069 because midkey is the same as first or last row 2024-11-26T23:26:11,225 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-26T23:26:11,225 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:11,225 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/d35daee0dc594d3596f2e837ca095069 because midkey is the same as first or last row 2024-11-26T23:26:11,226 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=27.1 K, sizeToCheck=16.0 K 2024-11-26T23:26:11,226 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:11,226 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/d35daee0dc594d3596f2e837ca095069 because midkey is the same as first or last row 2024-11-26T23:26:11,226 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:26:11,226 DEBUG [RS:0;c707e9ace675:37045-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: fb3bce74abd382c981bf9c89508ce7a0:info 2024-11-26T23:26:11,510 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/WALs/c707e9ace675,37045,1732663496045/c707e9ace675%2C37045%2C1732663496045.1732663565627 to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/oldWALs/c707e9ace675%2C37045%2C1732663496045.1732663565627 2024-11-26T23:26:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37045 {}] regionserver.HRegion(8855): Flush requested on fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:26:23,136 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing fb3bce74abd382c981bf9c89508ce7a0 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-26T23:26:23,143 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/7e2702422d6741048da2870f6d3d8147 is 1080, key is row0022/info:/1732663571110/Put/seqid=0 2024-11-26T23:26:23,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741849_1025 (size=12509) 2024-11-26T23:26:23,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741849_1025 (size=12509) 2024-11-26T23:26:23,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/7e2702422d6741048da2870f6d3d8147 2024-11-26T23:26:23,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/7e2702422d6741048da2870f6d3d8147 as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/7e2702422d6741048da2870f6d3d8147 2024-11-26T23:26:23,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/7e2702422d6741048da2870f6d3d8147, entries=7, sequenceid=42, filesize=12.2 K 2024-11-26T23:26:23,172 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=0 B/0 for fb3bce74abd382c981bf9c89508ce7a0 in 36ms, sequenceid=42, compaction requested=false 2024-11-26T23:26:23,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for fb3bce74abd382c981bf9c89508ce7a0: 2024-11-26T23:26:23,172 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.3 K, sizeToCheck=16.0 K 2024-11-26T23:26:23,172 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:23,172 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/d35daee0dc594d3596f2e837ca095069 because midkey is the same as first or last row 2024-11-26T23:26:24,430 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:26:28,566 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region fb3bce74abd382c981bf9c89508ce7a0, had cached 0 bytes from a total of 40219 2024-11-26T23:26:31,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T23:26:31,148 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:26:31,148 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:26:31,153 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:31,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:31,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:26:31,154 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T23:26:31,154 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=278837223, stopped=false 2024-11-26T23:26:31,155 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c707e9ace675,34413,1732663495322 2024-11-26T23:26:31,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:26:31,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:31,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:26:31,157 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:26:31,157 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:31,157 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:26:31,157 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:26:31,157 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:31,158 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:26:31,158 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,37045,1732663496045' ***** 2024-11-26T23:26:31,158 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T23:26:31,158 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:26:31,158 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:26:31,158 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:26:31,159 INFO [RS:0;c707e9ace675:37045 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T23:26:31,159 INFO [RS:0;c707e9ace675:37045 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T23:26:31,159 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(3091): Received CLOSE for fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:26:31,159 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(959): stopping server c707e9ace675,37045,1732663496045 2024-11-26T23:26:31,159 INFO [RS:0;c707e9ace675:37045 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:26:31,159 INFO [RS:0;c707e9ace675:37045 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c707e9ace675:37045. 2024-11-26T23:26:31,160 DEBUG [RS:0;c707e9ace675:37045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:26:31,160 DEBUG [RS:0;c707e9ace675:37045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:31,160 INFO [RS:0;c707e9ace675:37045 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:26:31,160 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb3bce74abd382c981bf9c89508ce7a0, disabling compactions & flushes 2024-11-26T23:26:31,160 INFO [RS:0;c707e9ace675:37045 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:26:31,160 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:26:31,160 INFO [RS:0;c707e9ace675:37045 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:26:31,160 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:26:31,160 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. after waiting 0 ms 2024-11-26T23:26:31,160 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T23:26:31,160 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:26:31,160 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing fb3bce74abd382c981bf9c89508ce7a0 1/1 column families, dataSize=3.15 KB heapSize=3.63 KB 2024-11-26T23:26:31,160 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T23:26:31,161 DEBUG [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, fb3bce74abd382c981bf9c89508ce7a0=TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.} 2024-11-26T23:26:31,160 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:26:31,161 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:26:31,161 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:26:31,161 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:26:31,161 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:26:31,161 DEBUG [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, fb3bce74abd382c981bf9c89508ce7a0 2024-11-26T23:26:31,161 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.65 KB heapSize=3.67 KB 2024-11-26T23:26:31,166 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/22bd9218eb394abab8d492893839fd27 is 1080, key is row0029/info:/1732663585138/Put/seqid=0 2024-11-26T23:26:31,169 INFO [regionserver/c707e9ace675:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:26:31,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741850_1026 (size=8193) 2024-11-26T23:26:31,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741850_1026 (size=8193) 2024-11-26T23:26:31,175 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=3.15 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/22bd9218eb394abab8d492893839fd27 2024-11-26T23:26:31,183 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/.tmp/info/b117bb3e2a30422c84bbf7672f232c1c is 195, key is TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0./info:regioninfo/1732663498589/Put/seqid=0 2024-11-26T23:26:31,183 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/.tmp/info/22bd9218eb394abab8d492893839fd27 as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/22bd9218eb394abab8d492893839fd27 2024-11-26T23:26:31,184 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T23:26:31,185 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T23:26:31,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741851_1027 (size=7016) 2024-11-26T23:26:31,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741851_1027 (size=7016) 2024-11-26T23:26:31,191 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.45 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/.tmp/info/b117bb3e2a30422c84bbf7672f232c1c 2024-11-26T23:26:31,192 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/22bd9218eb394abab8d492893839fd27, entries=3, sequenceid=48, filesize=8.0 K 2024-11-26T23:26:31,193 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for fb3bce74abd382c981bf9c89508ce7a0 in 33ms, sequenceid=48, compaction requested=true 2024-11-26T23:26:31,194 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/07b58ef5960549cc85f858315ca0cbbb, hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/744436871fba47948dc671164cb7417f, hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/86f4f67398e946638aa4d9be6b71aacd] to archive 2024-11-26T23:26:31,197 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:26:31,200 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/07b58ef5960549cc85f858315ca0cbbb to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/archive/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/07b58ef5960549cc85f858315ca0cbbb 2024-11-26T23:26:31,202 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/744436871fba47948dc671164cb7417f to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/archive/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/744436871fba47948dc671164cb7417f 2024-11-26T23:26:31,204 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/86f4f67398e946638aa4d9be6b71aacd to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/archive/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/info/86f4f67398e946638aa4d9be6b71aacd 2024-11-26T23:26:31,214 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/.tmp/ns/f95bd7b8e3c84372a1615bf4287acf47 is 43, key is default/ns:d/1732663497889/Put/seqid=0 2024-11-26T23:26:31,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741852_1028 (size=5153) 2024-11-26T23:26:31,216 DEBUG [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c707e9ace675:34413 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-26T23:26:31,221 WARN [StoreCloser-TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [07b58ef5960549cc85f858315ca0cbbb=12509, 744436871fba47948dc671164cb7417f=12509, 86f4f67398e946638aa4d9be6b71aacd=12509] 2024-11-26T23:26:31,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741852_1028 (size=5153) 2024-11-26T23:26:31,222 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/.tmp/ns/f95bd7b8e3c84372a1615bf4287acf47 2024-11-26T23:26:31,229 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/default/TestLogRolling-testSlowSyncLogRolling/fb3bce74abd382c981bf9c89508ce7a0/recovered.edits/51.seqid, newMaxSeqId=51, maxSeqId=1 2024-11-26T23:26:31,231 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:26:31,232 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb3bce74abd382c981bf9c89508ce7a0: Waiting for close lock at 1732663591159Running coprocessor pre-close hooks at 1732663591160 (+1 ms)Disabling compacts and flushes for region at 1732663591160Disabling writes for close at 1732663591160Obtaining lock to block concurrent updates at 1732663591160Preparing flush snapshotting stores in fb3bce74abd382c981bf9c89508ce7a0 at 1732663591160Finished memstore snapshotting TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0., syncing WAL and waiting on mvcc, flushsize=dataSize=3228, getHeapSize=3696, getOffHeapSize=0, getCellsCount=3 at 1732663591161 (+1 ms)Flushing stores of TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. at 1732663591162 (+1 ms)Flushing fb3bce74abd382c981bf9c89508ce7a0/info: creating writer at 1732663591162Flushing fb3bce74abd382c981bf9c89508ce7a0/info: appending metadata at 1732663591166 (+4 ms)Flushing fb3bce74abd382c981bf9c89508ce7a0/info: closing flushed file at 1732663591166Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5679da1b: reopening flushed file at 1732663591182 (+16 ms)Finished flush of dataSize ~3.15 KB/3228, heapSize ~3.61 KB/3696, currentSize=0 B/0 for fb3bce74abd382c981bf9c89508ce7a0 in 33ms, sequenceid=48, compaction requested=true at 1732663591194 (+12 ms)Writing region close event to WAL at 1732663591224 (+30 ms)Running coprocessor post-close hooks at 1732663591229 (+5 ms)Closed at 1732663591231 (+2 ms) 2024-11-26T23:26:31,232 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testSlowSyncLogRolling,,1732663498140.fb3bce74abd382c981bf9c89508ce7a0. 2024-11-26T23:26:31,246 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/.tmp/table/944631f86ab24a59953b46f4f163373b is 73, key is TestLogRolling-testSlowSyncLogRolling/table:state/1732663498605/Put/seqid=0 2024-11-26T23:26:31,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741853_1029 (size=5396) 2024-11-26T23:26:31,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741853_1029 (size=5396) 2024-11-26T23:26:31,253 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=138 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/.tmp/table/944631f86ab24a59953b46f4f163373b 2024-11-26T23:26:31,260 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/.tmp/info/b117bb3e2a30422c84bbf7672f232c1c as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/info/b117bb3e2a30422c84bbf7672f232c1c 2024-11-26T23:26:31,267 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/info/b117bb3e2a30422c84bbf7672f232c1c, entries=10, sequenceid=11, filesize=6.9 K 2024-11-26T23:26:31,269 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/.tmp/ns/f95bd7b8e3c84372a1615bf4287acf47 as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/ns/f95bd7b8e3c84372a1615bf4287acf47 2024-11-26T23:26:31,276 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/ns/f95bd7b8e3c84372a1615bf4287acf47, entries=2, sequenceid=11, filesize=5.0 K 2024-11-26T23:26:31,277 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/.tmp/table/944631f86ab24a59953b46f4f163373b as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/table/944631f86ab24a59953b46f4f163373b 2024-11-26T23:26:31,283 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/table/944631f86ab24a59953b46f4f163373b, entries=2, sequenceid=11, filesize=5.3 K 2024-11-26T23:26:31,285 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 124ms, sequenceid=11, compaction requested=false 2024-11-26T23:26:31,291 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-26T23:26:31,292 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:26:31,292 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:26:31,292 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663591160Running coprocessor pre-close hooks at 1732663591160Disabling compacts and flushes for region at 1732663591160Disabling writes for close at 1732663591161 (+1 ms)Obtaining lock to block concurrent updates at 1732663591161Preparing flush snapshotting stores in 1588230740 at 1732663591161Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1692, getHeapSize=3696, getOffHeapSize=0, getCellsCount=14 at 1732663591161Flushing stores of hbase:meta,,1.1588230740 at 1732663591162 (+1 ms)Flushing 1588230740/info: creating writer at 1732663591163 (+1 ms)Flushing 1588230740/info: appending metadata at 1732663591182 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732663591182Flushing 1588230740/ns: creating writer at 1732663591198 (+16 ms)Flushing 1588230740/ns: appending metadata at 1732663591213 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732663591213Flushing 1588230740/table: creating writer at 1732663591229 (+16 ms)Flushing 1588230740/table: appending metadata at 1732663591245 (+16 ms)Flushing 1588230740/table: closing flushed file at 1732663591245Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e1e4953: reopening flushed file at 1732663591259 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@322ffdbb: reopening flushed file at 1732663591268 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bd64d5c: reopening flushed file at 1732663591276 (+8 ms)Finished flush of dataSize ~1.65 KB/1692, heapSize ~3.38 KB/3456, currentSize=0 B/0 for 1588230740 in 124ms, sequenceid=11, compaction requested=false at 1732663591285 (+9 ms)Writing region close event to WAL at 1732663591286 (+1 ms)Running coprocessor post-close hooks at 1732663591292 (+6 ms)Closed at 1732663591292 2024-11-26T23:26:31,292 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T23:26:31,361 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,37045,1732663496045; all regions closed. 2024-11-26T23:26:31,363 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,363 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,363 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,363 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,363 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741834_1010 (size=3066) 2024-11-26T23:26:31,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741834_1010 (size=3066) 2024-11-26T23:26:31,369 DEBUG [RS:0;c707e9ace675:37045 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/oldWALs 2024-11-26T23:26:31,369 INFO [RS:0;c707e9ace675:37045 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C37045%2C1732663496045.meta:.meta(num 1732663497742) 2024-11-26T23:26:31,370 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,370 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,370 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,370 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,370 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741847_1023 (size=12695) 2024-11-26T23:26:31,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741847_1023 (size=12695) 2024-11-26T23:26:31,376 DEBUG [RS:0;c707e9ace675:37045 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/oldWALs 2024-11-26T23:26:31,376 INFO [RS:0;c707e9ace675:37045 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C37045%2C1732663496045:(num 1732663571108) 2024-11-26T23:26:31,376 DEBUG [RS:0;c707e9ace675:37045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:31,376 INFO [RS:0;c707e9ace675:37045 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:26:31,376 INFO [RS:0;c707e9ace675:37045 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:26:31,376 INFO [RS:0;c707e9ace675:37045 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T23:26:31,377 INFO [RS:0;c707e9ace675:37045 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:26:31,377 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:26:31,377 INFO [RS:0;c707e9ace675:37045 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37045 2024-11-26T23:26:31,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,37045,1732663496045 2024-11-26T23:26:31,381 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:26:31,381 INFO [RS:0;c707e9ace675:37045 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:26:31,384 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,37045,1732663496045] 2024-11-26T23:26:31,386 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,37045,1732663496045 already deleted, retry=false 2024-11-26T23:26:31,386 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c707e9ace675,37045,1732663496045 expired; onlineServers=0 2024-11-26T23:26:31,386 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c707e9ace675,34413,1732663495322' ***** 2024-11-26T23:26:31,386 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T23:26:31,386 INFO [M:0;c707e9ace675:34413 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:26:31,386 INFO [M:0;c707e9ace675:34413 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:26:31,386 DEBUG [M:0;c707e9ace675:34413 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T23:26:31,387 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T23:26:31,387 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663496999 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663496999,5,FailOnTimeoutGroup] 2024-11-26T23:26:31,387 DEBUG [M:0;c707e9ace675:34413 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T23:26:31,387 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663497000 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663497000,5,FailOnTimeoutGroup] 2024-11-26T23:26:31,387 INFO [M:0;c707e9ace675:34413 {}] hbase.ChoreService(370): Chore service for: master/c707e9ace675:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T23:26:31,387 INFO [M:0;c707e9ace675:34413 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:26:31,387 DEBUG [M:0;c707e9ace675:34413 {}] master.HMaster(1795): Stopping service threads 2024-11-26T23:26:31,387 INFO [M:0;c707e9ace675:34413 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T23:26:31,387 INFO [M:0;c707e9ace675:34413 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:26:31,388 INFO [M:0;c707e9ace675:34413 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T23:26:31,388 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T23:26:31,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T23:26:31,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:31,389 DEBUG [M:0;c707e9ace675:34413 {}] zookeeper.ZKUtil(347): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T23:26:31,389 WARN [M:0;c707e9ace675:34413 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T23:26:31,390 INFO [M:0;c707e9ace675:34413 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/.lastflushedseqids 2024-11-26T23:26:31,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741854_1030 (size=130) 2024-11-26T23:26:31,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741854_1030 (size=130) 2024-11-26T23:26:31,402 INFO [M:0;c707e9ace675:34413 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T23:26:31,403 INFO [M:0;c707e9ace675:34413 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T23:26:31,403 DEBUG [M:0;c707e9ace675:34413 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:26:31,403 INFO [M:0;c707e9ace675:34413 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:31,403 DEBUG [M:0;c707e9ace675:34413 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:31,403 DEBUG [M:0;c707e9ace675:34413 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:26:31,403 DEBUG [M:0;c707e9ace675:34413 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:31,403 INFO [M:0;c707e9ace675:34413 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.04 KB heapSize=29.21 KB 2024-11-26T23:26:31,422 DEBUG [M:0;c707e9ace675:34413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a8f6ce6ec51b46c6930ccc8bdd1c7953 is 82, key is hbase:meta,,1/info:regioninfo/1732663497815/Put/seqid=0 2024-11-26T23:26:31,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741855_1031 (size=5672) 2024-11-26T23:26:31,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741855_1031 (size=5672) 2024-11-26T23:26:31,428 INFO [M:0;c707e9ace675:34413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a8f6ce6ec51b46c6930ccc8bdd1c7953 2024-11-26T23:26:31,451 DEBUG [M:0;c707e9ace675:34413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/baecee2b183548d4b96d076fdfc3c21e is 767, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732663498612/Put/seqid=0 2024-11-26T23:26:31,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741856_1032 (size=6248) 2024-11-26T23:26:31,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741856_1032 (size=6248) 2024-11-26T23:26:31,457 INFO [M:0;c707e9ace675:34413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.43 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/baecee2b183548d4b96d076fdfc3c21e 2024-11-26T23:26:31,463 INFO [M:0;c707e9ace675:34413 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for baecee2b183548d4b96d076fdfc3c21e 2024-11-26T23:26:31,479 DEBUG [M:0;c707e9ace675:34413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b4cb2ced455f4f8ca444c1ae754d05c2 is 69, key is c707e9ace675,37045,1732663496045/rs:state/1732663497079/Put/seqid=0 2024-11-26T23:26:31,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:26:31,485 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37045-0x1015698b1e90001, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:26:31,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741857_1033 (size=5156) 2024-11-26T23:26:31,485 INFO [RS:0;c707e9ace675:37045 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:26:31,485 INFO [RS:0;c707e9ace675:37045 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,37045,1732663496045; zookeeper connection closed. 2024-11-26T23:26:31,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741857_1033 (size=5156) 2024-11-26T23:26:31,486 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1ee16e73 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1ee16e73 2024-11-26T23:26:31,486 INFO [M:0;c707e9ace675:34413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b4cb2ced455f4f8ca444c1ae754d05c2 2024-11-26T23:26:31,486 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T23:26:31,510 DEBUG [M:0;c707e9ace675:34413 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/eb9853c80a824d2291ac5ef105a14210 is 52, key is load_balancer_on/state:d/1732663498119/Put/seqid=0 2024-11-26T23:26:31,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741858_1034 (size=5056) 2024-11-26T23:26:31,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741858_1034 (size=5056) 2024-11-26T23:26:31,516 INFO [M:0;c707e9ace675:34413 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/eb9853c80a824d2291ac5ef105a14210 2024-11-26T23:26:31,523 DEBUG [M:0;c707e9ace675:34413 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/a8f6ce6ec51b46c6930ccc8bdd1c7953 as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a8f6ce6ec51b46c6930ccc8bdd1c7953 2024-11-26T23:26:31,531 INFO [M:0;c707e9ace675:34413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/a8f6ce6ec51b46c6930ccc8bdd1c7953, entries=8, sequenceid=59, filesize=5.5 K 2024-11-26T23:26:31,532 DEBUG [M:0;c707e9ace675:34413 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/baecee2b183548d4b96d076fdfc3c21e as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/baecee2b183548d4b96d076fdfc3c21e 2024-11-26T23:26:31,539 INFO [M:0;c707e9ace675:34413 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for baecee2b183548d4b96d076fdfc3c21e 2024-11-26T23:26:31,539 INFO [M:0;c707e9ace675:34413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/baecee2b183548d4b96d076fdfc3c21e, entries=6, sequenceid=59, filesize=6.1 K 2024-11-26T23:26:31,540 DEBUG [M:0;c707e9ace675:34413 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b4cb2ced455f4f8ca444c1ae754d05c2 as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b4cb2ced455f4f8ca444c1ae754d05c2 2024-11-26T23:26:31,547 INFO [M:0;c707e9ace675:34413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b4cb2ced455f4f8ca444c1ae754d05c2, entries=1, sequenceid=59, filesize=5.0 K 2024-11-26T23:26:31,548 DEBUG [M:0;c707e9ace675:34413 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/eb9853c80a824d2291ac5ef105a14210 as hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/eb9853c80a824d2291ac5ef105a14210 2024-11-26T23:26:31,555 INFO [M:0;c707e9ace675:34413 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/eb9853c80a824d2291ac5ef105a14210, entries=1, sequenceid=59, filesize=4.9 K 2024-11-26T23:26:31,556 INFO [M:0;c707e9ace675:34413 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 153ms, sequenceid=59, compaction requested=false 2024-11-26T23:26:31,558 INFO [M:0;c707e9ace675:34413 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:31,558 DEBUG [M:0;c707e9ace675:34413 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663591403Disabling compacts and flushes for region at 1732663591403Disabling writes for close at 1732663591403Obtaining lock to block concurrent updates at 1732663591403Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732663591403Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23588, getHeapSize=29848, getOffHeapSize=0, getCellsCount=70 at 1732663591404 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732663591404Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732663591405 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732663591421 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732663591421Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732663591434 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732663591450 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732663591451 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732663591463 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732663591479 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732663591479Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732663591493 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732663591509 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732663591509Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@543c9417: reopening flushed file at 1732663591522 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b559652: reopening flushed file at 1732663591531 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6fda33f9: reopening flushed file at 1732663591539 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b9c9deb: reopening flushed file at 1732663591547 (+8 ms)Finished flush of dataSize ~23.04 KB/23588, heapSize ~29.15 KB/29848, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 153ms, sequenceid=59, compaction requested=false at 1732663591556 (+9 ms)Writing region close event to WAL at 1732663591558 (+2 ms)Closed at 1732663591558 2024-11-26T23:26:31,559 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,559 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,559 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,559 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,559 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:31,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36577 is added to blk_1073741830_1006 (size=27985) 2024-11-26T23:26:31,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41255 is added to blk_1073741830_1006 (size=27985) 2024-11-26T23:26:31,562 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:26:31,562 INFO [M:0;c707e9ace675:34413 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T23:26:31,562 INFO [M:0;c707e9ace675:34413 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34413 2024-11-26T23:26:31,563 INFO [M:0;c707e9ace675:34413 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:26:31,665 INFO [M:0;c707e9ace675:34413 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:26:31,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:26:31,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34413-0x1015698b1e90000, quorum=127.0.0.1:59934, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:26:31,670 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bf97579{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:31,672 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@22b88bcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:26:31,672 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:26:31,672 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d48d695{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:26:31,672 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11effdcd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/hadoop.log.dir/,STOPPED} 2024-11-26T23:26:31,675 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:26:31,675 WARN [BP-2092199251-172.17.0.2-1732663492556 heartbeating to localhost/127.0.0.1:34219 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:26:31,675 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:26:31,675 WARN [BP-2092199251-172.17.0.2-1732663492556 heartbeating to localhost/127.0.0.1:34219 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2092199251-172.17.0.2-1732663492556 (Datanode Uuid 533e0587-e020-45bd-b4e4-dfd9c6fc0426) service to localhost/127.0.0.1:34219 2024-11-26T23:26:31,677 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db/data/data3/current/BP-2092199251-172.17.0.2-1732663492556 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:31,677 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db/data/data4/current/BP-2092199251-172.17.0.2-1732663492556 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:31,678 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:26:31,682 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7b07d1ba{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:31,682 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@43e0a762{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:26:31,682 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:26:31,682 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e191c{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:26:31,682 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@28778f0f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/hadoop.log.dir/,STOPPED} 2024-11-26T23:26:31,684 WARN [BP-2092199251-172.17.0.2-1732663492556 heartbeating to localhost/127.0.0.1:34219 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:26:31,684 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:26:31,684 WARN [BP-2092199251-172.17.0.2-1732663492556 heartbeating to localhost/127.0.0.1:34219 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2092199251-172.17.0.2-1732663492556 (Datanode Uuid ff3656cd-c96f-4a2f-a2bd-e7e16af0dd78) service to localhost/127.0.0.1:34219 2024-11-26T23:26:31,684 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:26:31,685 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db/data/data1/current/BP-2092199251-172.17.0.2-1732663492556 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:31,685 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/cluster_8a6854bc-4f8d-7ae0-52c7-a24d78b6d1db/data/data2/current/BP-2092199251-172.17.0.2-1732663492556 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:31,685 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:26:31,694 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@735fa16a{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:26:31,695 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c26a5a3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:26:31,695 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:26:31,695 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70be1389{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:26:31,695 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@ddc8467{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/hadoop.log.dir/,STOPPED} 2024-11-26T23:26:31,704 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T23:26:31,739 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T23:26:31,750 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testSlowSyncLogRolling Thread=78 (was 12) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Idle-Rpc-Conn-Sweeper-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34219 from jenkins.hfs.0 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34219 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Time-limited test.named-queue-events-pool-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34219 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SSL Certificates Store Monitor java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ForkJoinPool-2-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-3-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SessionTracker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: regionserver/c707e9ace675:0.procedureResultReporter java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) Potentially hanging thread: region-location-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-3-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-5-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async-Client-Retry-Timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool-2-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-2-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.hdfs.PeerCache@221b72a4 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34219 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-3-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: master/c707e9ace675:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:34219 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-2-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: SnapshotHandlerChoreCleaner java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: nioEventLoopGroup-4-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.0@localhost:34219 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-5-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34219 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34219 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: ForkJoinPool-2-worker-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: nioEventLoopGroup-5-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-4-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: ForkJoinPool-2-worker-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: master/c707e9ace675:0:becomeActiveMaster-MemStoreChunkPool Statistics java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: HMaster-EventLoopGroup-1-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RpcClient-timer-pool-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=402 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=204 (was 304), ProcessCount=11 (was 11), AvailableMemoryMB=6779 (was 7382) 2024-11-26T23:26:31,757 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=79, OpenFileDescriptor=402, MaxFileDescriptor=1048576, SystemLoadAverage=204, ProcessCount=11, AvailableMemoryMB=6778 2024-11-26T23:26:31,757 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T23:26:31,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/hadoop.log.dir so I do NOT create it in target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206 2024-11-26T23:26:31,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/49217758-37af-ba7d-13ed-e5a2643c5c3c/hadoop.tmp.dir so I do NOT create it in target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206 2024-11-26T23:26:31,758 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22, deleteOnExit=true 2024-11-26T23:26:31,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T23:26:31,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/test.cache.data in system properties and HBase conf 2024-11-26T23:26:31,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T23:26:31,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/hadoop.log.dir in system properties and HBase conf 2024-11-26T23:26:31,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T23:26:31,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T23:26:31,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T23:26:31,759 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T23:26:31,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:26:31,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:26:31,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T23:26:31,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:26:31,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T23:26:31,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T23:26:31,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:26:31,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:26:31,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T23:26:31,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/nfs.dump.dir in system properties and HBase conf 2024-11-26T23:26:31,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/java.io.tmpdir in system properties and HBase conf 2024-11-26T23:26:31,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:26:31,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T23:26:31,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T23:26:31,776 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:26:31,859 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:31,865 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:26:31,866 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:26:31,866 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:26:31,866 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:26:31,867 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:31,868 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1ae773f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:26:31,868 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@630e1a46{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:26:31,986 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d195093{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/java.io.tmpdir/jetty-localhost-44599-hadoop-hdfs-3_4_1-tests_jar-_-any-17557360026235723831/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:26:31,987 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@476d81{HTTP/1.1, (http/1.1)}{localhost:44599} 2024-11-26T23:26:31,987 INFO [Time-limited test {}] server.Server(415): Started @101332ms 2024-11-26T23:26:32,002 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:26:32,081 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:32,086 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:26:32,087 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:26:32,087 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:26:32,087 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:26:32,088 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@670e4080{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:26:32,089 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7247ee1d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:26:32,209 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6a249094{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/java.io.tmpdir/jetty-localhost-45135-hadoop-hdfs-3_4_1-tests_jar-_-any-10524182716147784851/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:32,210 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@35445cbd{HTTP/1.1, (http/1.1)}{localhost:45135} 2024-11-26T23:26:32,210 INFO [Time-limited test {}] server.Server(415): Started @101555ms 2024-11-26T23:26:32,212 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:26:32,255 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:32,259 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:26:32,259 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:26:32,259 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:26:32,260 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:26:32,260 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@18f27499{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:26:32,260 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@64e5ce98{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:26:32,348 WARN [Thread-438 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22/data/data2/current/BP-1251012957-172.17.0.2-1732663591795/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:32,348 WARN [Thread-437 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22/data/data1/current/BP-1251012957-172.17.0.2-1732663591795/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:32,366 WARN [Thread-416 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:26:32,369 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf57c18f2ff0bebde with lease ID 0x155d7551a1ad2450: Processing first storage report for DS-fb712dbd-fc8f-419b-b5ed-2a9835f0b222 from datanode DatanodeRegistration(127.0.0.1:44775, datanodeUuid=5b120619-752d-4b9e-b6e7-9daf2f10513c, infoPort=33191, infoSecurePort=0, ipcPort=42271, storageInfo=lv=-57;cid=testClusterID;nsid=2140943674;c=1732663591795) 2024-11-26T23:26:32,369 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf57c18f2ff0bebde with lease ID 0x155d7551a1ad2450: from storage DS-fb712dbd-fc8f-419b-b5ed-2a9835f0b222 node DatanodeRegistration(127.0.0.1:44775, datanodeUuid=5b120619-752d-4b9e-b6e7-9daf2f10513c, infoPort=33191, infoSecurePort=0, ipcPort=42271, storageInfo=lv=-57;cid=testClusterID;nsid=2140943674;c=1732663591795), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:32,369 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf57c18f2ff0bebde with lease ID 0x155d7551a1ad2450: Processing first storage report for DS-552c80b6-e4db-4256-892b-f3f3333b6375 from datanode DatanodeRegistration(127.0.0.1:44775, datanodeUuid=5b120619-752d-4b9e-b6e7-9daf2f10513c, infoPort=33191, infoSecurePort=0, ipcPort=42271, storageInfo=lv=-57;cid=testClusterID;nsid=2140943674;c=1732663591795) 2024-11-26T23:26:32,369 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf57c18f2ff0bebde with lease ID 0x155d7551a1ad2450: from storage DS-552c80b6-e4db-4256-892b-f3f3333b6375 node DatanodeRegistration(127.0.0.1:44775, datanodeUuid=5b120619-752d-4b9e-b6e7-9daf2f10513c, infoPort=33191, infoSecurePort=0, ipcPort=42271, storageInfo=lv=-57;cid=testClusterID;nsid=2140943674;c=1732663591795), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:32,380 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@254b7745{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/java.io.tmpdir/jetty-localhost-39371-hadoop-hdfs-3_4_1-tests_jar-_-any-12360641021807761271/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:32,381 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@86e294e{HTTP/1.1, (http/1.1)}{localhost:39371} 2024-11-26T23:26:32,381 INFO [Time-limited test {}] server.Server(415): Started @101726ms 2024-11-26T23:26:32,382 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:26:32,482 WARN [Thread-463 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22/data/data3/current/BP-1251012957-172.17.0.2-1732663591795/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:32,482 WARN [Thread-464 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22/data/data4/current/BP-1251012957-172.17.0.2-1732663591795/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:32,502 WARN [Thread-452 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:26:32,505 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa80d52d077693a61 with lease ID 0x155d7551a1ad2451: Processing first storage report for DS-d9fcaa21-ade9-473b-9ceb-62524d4cb5bf from datanode DatanodeRegistration(127.0.0.1:35705, datanodeUuid=64d714f4-4608-4957-b402-a0653bb85756, infoPort=41825, infoSecurePort=0, ipcPort=33667, storageInfo=lv=-57;cid=testClusterID;nsid=2140943674;c=1732663591795) 2024-11-26T23:26:32,505 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa80d52d077693a61 with lease ID 0x155d7551a1ad2451: from storage DS-d9fcaa21-ade9-473b-9ceb-62524d4cb5bf node DatanodeRegistration(127.0.0.1:35705, datanodeUuid=64d714f4-4608-4957-b402-a0653bb85756, infoPort=41825, infoSecurePort=0, ipcPort=33667, storageInfo=lv=-57;cid=testClusterID;nsid=2140943674;c=1732663591795), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:32,506 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xa80d52d077693a61 with lease ID 0x155d7551a1ad2451: Processing first storage report for DS-0ea2852c-6dc4-4b43-aaf8-7c27c95265e8 from datanode DatanodeRegistration(127.0.0.1:35705, datanodeUuid=64d714f4-4608-4957-b402-a0653bb85756, infoPort=41825, infoSecurePort=0, ipcPort=33667, storageInfo=lv=-57;cid=testClusterID;nsid=2140943674;c=1732663591795) 2024-11-26T23:26:32,506 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa80d52d077693a61 with lease ID 0x155d7551a1ad2451: from storage DS-0ea2852c-6dc4-4b43-aaf8-7c27c95265e8 node DatanodeRegistration(127.0.0.1:35705, datanodeUuid=64d714f4-4608-4957-b402-a0653bb85756, infoPort=41825, infoSecurePort=0, ipcPort=33667, storageInfo=lv=-57;cid=testClusterID;nsid=2140943674;c=1732663591795), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:32,510 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206 2024-11-26T23:26:32,512 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22/zookeeper_0, clientPort=63702, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T23:26:32,513 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=63702 2024-11-26T23:26:32,514 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:32,515 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:32,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:26:32,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:26:32,527 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f with version=8 2024-11-26T23:26:32,527 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/hbase-staging 2024-11-26T23:26:32,529 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:26:32,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:32,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:32,530 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:26:32,530 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:32,530 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:26:32,530 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T23:26:32,530 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:26:32,531 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44407 2024-11-26T23:26:32,532 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44407 connecting to ZooKeeper ensemble=127.0.0.1:63702 2024-11-26T23:26:32,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:444070x0, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:26:32,544 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44407-0x101569a30cf0000 connected 2024-11-26T23:26:32,559 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:32,561 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:32,563 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:26:32,563 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f, hbase.cluster.distributed=false 2024-11-26T23:26:32,565 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:26:32,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44407 2024-11-26T23:26:32,565 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44407 2024-11-26T23:26:32,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44407 2024-11-26T23:26:32,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44407 2024-11-26T23:26:32,566 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44407 2024-11-26T23:26:32,583 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:26:32,583 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:32,583 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:32,583 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:26:32,583 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:32,583 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:26:32,583 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:26:32,583 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:26:32,584 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41115 2024-11-26T23:26:32,585 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41115 connecting to ZooKeeper ensemble=127.0.0.1:63702 2024-11-26T23:26:32,586 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:32,588 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:32,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:411150x0, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:26:32,593 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:411150x0, quorum=127.0.0.1:63702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:26:32,593 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41115-0x101569a30cf0001 connected 2024-11-26T23:26:32,593 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:26:32,594 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:26:32,595 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T23:26:32,596 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:26:32,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41115 2024-11-26T23:26:32,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41115 2024-11-26T23:26:32,601 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41115 2024-11-26T23:26:32,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41115 2024-11-26T23:26:32,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41115 2024-11-26T23:26:32,618 DEBUG [M:0;c707e9ace675:44407 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c707e9ace675:44407 2024-11-26T23:26:32,618 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c707e9ace675,44407,1732663592529 2024-11-26T23:26:32,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:26:32,620 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:26:32,621 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c707e9ace675,44407,1732663592529 2024-11-26T23:26:32,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T23:26:32,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:32,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:32,623 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T23:26:32,624 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c707e9ace675,44407,1732663592529 from backup master directory 2024-11-26T23:26:32,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c707e9ace675,44407,1732663592529 2024-11-26T23:26:32,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:26:32,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:26:32,626 WARN [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:26:32,626 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c707e9ace675,44407,1732663592529 2024-11-26T23:26:32,632 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/hbase.id] with ID: cbf7212b-70f8-401a-8ed7-261de5a6fe05 2024-11-26T23:26:32,632 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/.tmp/hbase.id 2024-11-26T23:26:32,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:26:32,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:26:32,640 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/.tmp/hbase.id]:[hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/hbase.id] 2024-11-26T23:26:32,657 INFO [master/c707e9ace675:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:32,657 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T23:26:32,659 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-26T23:26:32,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:32,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:32,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:26:32,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:26:32,672 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:26:32,673 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T23:26:32,676 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:26:32,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:26:32,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:26:32,687 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store 2024-11-26T23:26:32,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:26:32,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:26:32,695 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:26:32,695 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:26:32,695 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:32,695 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:32,695 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:26:32,695 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:32,695 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:32,696 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663592695Disabling compacts and flushes for region at 1732663592695Disabling writes for close at 1732663592695Writing region close event to WAL at 1732663592695Closed at 1732663592695 2024-11-26T23:26:32,697 WARN [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/.initializing 2024-11-26T23:26:32,697 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/WALs/c707e9ace675,44407,1732663592529 2024-11-26T23:26:32,700 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C44407%2C1732663592529, suffix=, logDir=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/WALs/c707e9ace675,44407,1732663592529, archiveDir=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/oldWALs, maxLogs=10 2024-11-26T23:26:32,700 INFO [master/c707e9ace675:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C44407%2C1732663592529.1732663592700 2024-11-26T23:26:32,706 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/WALs/c707e9ace675,44407,1732663592529/c707e9ace675%2C44407%2C1732663592529.1732663592700 2024-11-26T23:26:32,706 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41825:41825),(127.0.0.1/127.0.0.1:33191:33191)] 2024-11-26T23:26:32,707 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:26:32,707 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:26:32,707 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,707 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,709 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T23:26:32,711 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:32,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:32,712 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,713 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T23:26:32,713 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:32,714 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:26:32,714 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T23:26:32,717 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:32,717 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:26:32,718 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T23:26:32,719 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:32,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:26:32,720 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,721 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,721 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,724 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,724 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,724 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:26:32,726 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:32,730 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:26:32,731 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=743479, jitterRate=-0.05461794137954712}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:26:32,733 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732663592708Initializing all the Stores at 1732663592709 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663592709Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663592709Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663592709Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663592709Cleaning up temporary data from old regions at 1732663592724 (+15 ms)Region opened successfully at 1732663592732 (+8 ms) 2024-11-26T23:26:32,733 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T23:26:32,739 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8804d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:26:32,740 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T23:26:32,740 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T23:26:32,740 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T23:26:32,741 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T23:26:32,741 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T23:26:32,742 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-26T23:26:32,742 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T23:26:32,747 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T23:26:32,748 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T23:26:32,749 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T23:26:32,750 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T23:26:32,750 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T23:26:32,752 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T23:26:32,752 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T23:26:32,757 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T23:26:32,764 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T23:26:32,765 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T23:26:32,766 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T23:26:32,769 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T23:26:32,770 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T23:26:32,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:26:32,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:26:32,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:32,772 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:32,773 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c707e9ace675,44407,1732663592529, sessionid=0x101569a30cf0000, setting cluster-up flag (Was=false) 2024-11-26T23:26:32,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:32,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:32,783 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T23:26:32,784 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,44407,1732663592529 2024-11-26T23:26:32,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:32,788 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:32,793 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T23:26:32,794 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,44407,1732663592529 2024-11-26T23:26:32,795 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T23:26:32,797 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T23:26:32,798 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T23:26:32,798 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T23:26:32,798 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c707e9ace675,44407,1732663592529 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T23:26:32,799 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:26:32,799 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:26:32,799 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:26:32,799 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:26:32,799 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c707e9ace675:0, corePoolSize=10, maxPoolSize=10 2024-11-26T23:26:32,799 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,800 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:26:32,800 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,800 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732663622800 2024-11-26T23:26:32,800 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T23:26:32,801 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T23:26:32,801 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T23:26:32,801 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T23:26:32,801 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T23:26:32,801 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T23:26:32,801 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,801 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T23:26:32,801 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T23:26:32,801 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T23:26:32,801 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:26:32,802 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T23:26:32,802 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T23:26:32,802 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T23:26:32,802 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663592802,5,FailOnTimeoutGroup] 2024-11-26T23:26:32,802 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663592802,5,FailOnTimeoutGroup] 2024-11-26T23:26:32,802 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,803 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T23:26:32,803 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,803 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,803 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:32,803 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T23:26:32,806 INFO [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(746): ClusterId : cbf7212b-70f8-401a-8ed7-261de5a6fe05 2024-11-26T23:26:32,806 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:26:32,808 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:26:32,808 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:26:32,811 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:26:32,811 DEBUG [RS:0;c707e9ace675:41115 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4abb679b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:26:32,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:26:32,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:26:32,814 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T23:26:32,814 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f 2024-11-26T23:26:32,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:26:32,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:26:32,828 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:26:32,829 DEBUG [RS:0;c707e9ace675:41115 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c707e9ace675:41115 2024-11-26T23:26:32,830 INFO [RS:0;c707e9ace675:41115 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:26:32,830 INFO [RS:0;c707e9ace675:41115 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:26:32,830 DEBUG [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:26:32,830 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:26:32,831 INFO [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,44407,1732663592529 with port=41115, startcode=1732663592583 2024-11-26T23:26:32,831 DEBUG [RS:0;c707e9ace675:41115 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:26:32,832 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:26:32,832 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:32,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:32,833 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:26:32,834 INFO [HMaster-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44849, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:26:32,834 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44407 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,41115,1732663592583 2024-11-26T23:26:32,835 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44407 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,41115,1732663592583 2024-11-26T23:26:32,835 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:26:32,835 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:32,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:32,836 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:26:32,837 DEBUG [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f 2024-11-26T23:26:32,837 DEBUG [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42807 2024-11-26T23:26:32,837 DEBUG [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:26:32,838 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:26:32,838 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:32,839 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:32,839 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:26:32,840 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:26:32,840 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:26:32,840 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:32,841 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:32,841 DEBUG [RS:0;c707e9ace675:41115 {}] zookeeper.ZKUtil(111): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,41115,1732663592583 2024-11-26T23:26:32,841 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:26:32,841 WARN [RS:0;c707e9ace675:41115 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:26:32,841 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,41115,1732663592583] 2024-11-26T23:26:32,842 INFO [RS:0;c707e9ace675:41115 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:26:32,842 DEBUG [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/WALs/c707e9ace675,41115,1732663592583 2024-11-26T23:26:32,842 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/1588230740 2024-11-26T23:26:32,843 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/1588230740 2024-11-26T23:26:32,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:26:32,845 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:26:32,846 INFO [RS:0;c707e9ace675:41115 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:26:32,846 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:26:32,847 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:26:32,850 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:26:32,850 INFO [RS:0;c707e9ace675:41115 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:26:32,851 INFO [RS:0;c707e9ace675:41115 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:26:32,851 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,851 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=792241, jitterRate=0.007387220859527588}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:26:32,851 INFO [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:26:32,852 INFO [RS:0;c707e9ace675:41115 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:26:32,852 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732663592828Initializing all the Stores at 1732663592829 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663592829Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663592830 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663592830Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663592830Cleaning up temporary data from old regions at 1732663592845 (+15 ms)Region opened successfully at 1732663592853 (+8 ms) 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:26:32,853 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:26:32,853 DEBUG [RS:0;c707e9ace675:41115 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:26:32,854 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:26:32,854 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663592853Disabling compacts and flushes for region at 1732663592853Disabling writes for close at 1732663592853Writing region close event to WAL at 1732663592854 (+1 ms)Closed at 1732663592854 2024-11-26T23:26:32,856 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:26:32,856 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T23:26:32,856 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T23:26:32,857 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,857 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,857 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,857 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,857 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,857 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,41115,1732663592583-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:26:32,858 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:26:32,860 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T23:26:32,883 INFO [RS:0;c707e9ace675:41115 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:26:32,883 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,41115,1732663592583-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,883 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,883 INFO [RS:0;c707e9ace675:41115 {}] regionserver.Replication(171): c707e9ace675,41115,1732663592583 started 2024-11-26T23:26:32,900 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:32,901 INFO [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,41115,1732663592583, RpcServer on c707e9ace675/172.17.0.2:41115, sessionid=0x101569a30cf0001 2024-11-26T23:26:32,901 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:26:32,901 DEBUG [RS:0;c707e9ace675:41115 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,41115,1732663592583 2024-11-26T23:26:32,901 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,41115,1732663592583' 2024-11-26T23:26:32,901 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:26:32,902 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:26:32,902 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:26:32,902 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:26:32,902 DEBUG [RS:0;c707e9ace675:41115 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,41115,1732663592583 2024-11-26T23:26:32,902 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,41115,1732663592583' 2024-11-26T23:26:32,902 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:26:32,903 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:26:32,903 DEBUG [RS:0;c707e9ace675:41115 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:26:32,903 INFO [RS:0;c707e9ace675:41115 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:26:32,903 INFO [RS:0;c707e9ace675:41115 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:26:33,006 INFO [RS:0;c707e9ace675:41115 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C41115%2C1732663592583, suffix=, logDir=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/WALs/c707e9ace675,41115,1732663592583, archiveDir=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/oldWALs, maxLogs=32 2024-11-26T23:26:33,008 INFO [RS:0;c707e9ace675:41115 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C41115%2C1732663592583.1732663593008 2024-11-26T23:26:33,011 WARN [c707e9ace675:44407 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T23:26:33,015 INFO [RS:0;c707e9ace675:41115 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/WALs/c707e9ace675,41115,1732663592583/c707e9ace675%2C41115%2C1732663592583.1732663593008 2024-11-26T23:26:33,016 DEBUG [RS:0;c707e9ace675:41115 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33191:33191),(127.0.0.1/127.0.0.1:41825:41825)] 2024-11-26T23:26:33,261 DEBUG [c707e9ace675:44407 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T23:26:33,262 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c707e9ace675,41115,1732663592583 2024-11-26T23:26:33,263 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,41115,1732663592583, state=OPENING 2024-11-26T23:26:33,265 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T23:26:33,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:33,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:33,267 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:26:33,267 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:26:33,267 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:26:33,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,41115,1732663592583}] 2024-11-26T23:26:33,422 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:26:33,424 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35417, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:26:33,429 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T23:26:33,429 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:26:33,431 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C41115%2C1732663592583.meta, suffix=.meta, logDir=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/WALs/c707e9ace675,41115,1732663592583, archiveDir=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/oldWALs, maxLogs=32 2024-11-26T23:26:33,433 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C41115%2C1732663592583.meta.1732663593433.meta 2024-11-26T23:26:33,439 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/WALs/c707e9ace675,41115,1732663592583/c707e9ace675%2C41115%2C1732663592583.meta.1732663593433.meta 2024-11-26T23:26:33,440 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:41825:41825),(127.0.0.1/127.0.0.1:33191:33191)] 2024-11-26T23:26:33,443 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:26:33,443 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T23:26:33,444 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T23:26:33,444 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T23:26:33,444 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T23:26:33,444 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:26:33,444 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T23:26:33,444 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T23:26:33,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:26:33,447 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:26:33,447 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:33,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:33,448 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:26:33,449 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:26:33,449 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:33,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:33,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:26:33,450 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:26:33,450 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:33,451 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:33,451 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:26:33,452 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:26:33,452 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:33,452 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:33,452 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:26:33,453 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/1588230740 2024-11-26T23:26:33,455 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/1588230740 2024-11-26T23:26:33,456 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:26:33,456 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:26:33,457 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:26:33,458 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:26:33,459 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=713845, jitterRate=-0.09229916334152222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:26:33,459 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T23:26:33,460 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732663593444Writing region info on filesystem at 1732663593444Initializing all the Stores at 1732663593446 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663593446Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663593446Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663593446Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663593446Cleaning up temporary data from old regions at 1732663593456 (+10 ms)Running coprocessor post-open hooks at 1732663593459 (+3 ms)Region opened successfully at 1732663593460 (+1 ms) 2024-11-26T23:26:33,462 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732663593421 2024-11-26T23:26:33,464 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T23:26:33,465 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T23:26:33,465 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,41115,1732663592583 2024-11-26T23:26:33,467 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,41115,1732663592583, state=OPEN 2024-11-26T23:26:33,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:26:33,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:26:33,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:26:33,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:26:33,472 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c707e9ace675,41115,1732663592583 2024-11-26T23:26:33,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T23:26:33,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,41115,1732663592583 in 204 msec 2024-11-26T23:26:33,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T23:26:33,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 619 msec 2024-11-26T23:26:33,480 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:26:33,480 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T23:26:33,481 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:26:33,481 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,41115,1732663592583, seqNum=-1] 2024-11-26T23:26:33,482 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:26:33,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60095, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:26:33,490 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 691 msec 2024-11-26T23:26:33,490 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732663593490, completionTime=-1 2024-11-26T23:26:33,490 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T23:26:33,490 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T23:26:33,492 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T23:26:33,492 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732663653492 2024-11-26T23:26:33,492 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732663713492 2024-11-26T23:26:33,492 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-26T23:26:33,492 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44407,1732663592529-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:33,492 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44407,1732663592529-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:33,492 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44407,1732663592529-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:33,492 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c707e9ace675:44407, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:33,493 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:33,493 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:33,495 DEBUG [master/c707e9ace675:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T23:26:33,498 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.871sec 2024-11-26T23:26:33,498 INFO [master/c707e9ace675:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T23:26:33,498 INFO [master/c707e9ace675:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T23:26:33,498 INFO [master/c707e9ace675:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T23:26:33,498 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T23:26:33,498 INFO [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T23:26:33,498 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44407,1732663592529-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:26:33,498 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44407,1732663592529-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T23:26:33,500 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T23:26:33,500 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T23:26:33,501 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44407,1732663592529-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:33,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fa3cb60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:26:33,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,44407,-1 for getting cluster id 2024-11-26T23:26:33,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:26:33,508 DEBUG [HMaster-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'cbf7212b-70f8-401a-8ed7-261de5a6fe05' 2024-11-26T23:26:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:26:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "cbf7212b-70f8-401a-8ed7-261de5a6fe05" 2024-11-26T23:26:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111b08da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:26:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,44407,-1] 2024-11-26T23:26:33,510 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:26:33,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:33,512 INFO [HMaster-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42288, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:26:33,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f14acfa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:26:33,513 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:26:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,41115,1732663592583, seqNum=-1] 2024-11-26T23:26:33,515 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:26:33,517 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-6-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:26:33,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c707e9ace675,44407,1732663592529 2024-11-26T23:26:33,520 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:33,524 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-26T23:26:33,525 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T23:26:33,525 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:26:33,525 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:26:33,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:33,525 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:33,525 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:26:33,526 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T23:26:33,526 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1469549761, stopped=false 2024-11-26T23:26:33,526 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c707e9ace675,44407,1732663592529 2024-11-26T23:26:33,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:26:33,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:26:33,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:33,528 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:33,528 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:26:33,529 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:26:33,529 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:26:33,529 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:26:33,529 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at org.apache.hadoop.hbase.regionserver.wal.TestLogRolling.testLogRollOnDatanodeDeath(TestLogRolling.java:201) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:26:33,530 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:33,531 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,41115,1732663592583' ***** 2024-11-26T23:26:33,531 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T23:26:33,532 INFO [RS:0;c707e9ace675:41115 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:26:33,533 INFO [RS:0;c707e9ace675:41115 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T23:26:33,533 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:26:33,533 INFO [RS:0;c707e9ace675:41115 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T23:26:33,533 INFO [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(959): stopping server c707e9ace675,41115,1732663592583 2024-11-26T23:26:33,533 INFO [RS:0;c707e9ace675:41115 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:26:33,533 INFO [RS:0;c707e9ace675:41115 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c707e9ace675:41115. 2024-11-26T23:26:33,533 DEBUG [RS:0;c707e9ace675:41115 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:26:33,533 DEBUG [RS:0;c707e9ace675:41115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:33,533 INFO [RS:0;c707e9ace675:41115 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:26:33,533 INFO [RS:0;c707e9ace675:41115 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:26:33,533 INFO [RS:0;c707e9ace675:41115 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:26:33,533 INFO [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T23:26:33,533 INFO [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T23:26:33,533 DEBUG [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T23:26:33,534 DEBUG [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:26:33,534 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:26:33,534 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:26:33,534 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:26:33,534 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:26:33,534 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:26:33,534 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-26T23:26:33,553 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/1588230740/.tmp/ns/ad65c0bead624422a5172b7b764f0f69 is 43, key is default/ns:d/1732663593484/Put/seqid=0 2024-11-26T23:26:33,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741835_1011 (size=5153) 2024-11-26T23:26:33,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741835_1011 (size=5153) 2024-11-26T23:26:33,560 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/1588230740/.tmp/ns/ad65c0bead624422a5172b7b764f0f69 2024-11-26T23:26:33,567 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/1588230740/.tmp/ns/ad65c0bead624422a5172b7b764f0f69 as hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/1588230740/ns/ad65c0bead624422a5172b7b764f0f69 2024-11-26T23:26:33,573 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/1588230740/ns/ad65c0bead624422a5172b7b764f0f69, entries=2, sequenceid=6, filesize=5.0 K 2024-11-26T23:26:33,575 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=6, compaction requested=false 2024-11-26T23:26:33,575 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T23:26:33,580 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-26T23:26:33,580 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:26:33,581 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:26:33,581 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663593534Running coprocessor pre-close hooks at 1732663593534Disabling compacts and flushes for region at 1732663593534Disabling writes for close at 1732663593534Obtaining lock to block concurrent updates at 1732663593534Preparing flush snapshotting stores in 1588230740 at 1732663593534Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732663593535 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732663593536 (+1 ms)Flushing 1588230740/ns: creating writer at 1732663593536Flushing 1588230740/ns: appending metadata at 1732663593553 (+17 ms)Flushing 1588230740/ns: closing flushed file at 1732663593553Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71bd53a8: reopening flushed file at 1732663593566 (+13 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 41ms, sequenceid=6, compaction requested=false at 1732663593575 (+9 ms)Writing region close event to WAL at 1732663593576 (+1 ms)Running coprocessor post-close hooks at 1732663593580 (+4 ms)Closed at 1732663593580 2024-11-26T23:26:33,581 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T23:26:33,734 INFO [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,41115,1732663592583; all regions closed. 2024-11-26T23:26:33,734 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,735 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,735 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,735 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,735 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741834_1010 (size=1152) 2024-11-26T23:26:33,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741834_1010 (size=1152) 2024-11-26T23:26:33,740 DEBUG [RS:0;c707e9ace675:41115 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/oldWALs 2024-11-26T23:26:33,740 INFO [RS:0;c707e9ace675:41115 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C41115%2C1732663592583.meta:.meta(num 1732663593433) 2024-11-26T23:26:33,741 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,741 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,741 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,741 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,741 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741833_1009 (size=93) 2024-11-26T23:26:33,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741833_1009 (size=93) 2024-11-26T23:26:33,746 DEBUG [RS:0;c707e9ace675:41115 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/oldWALs 2024-11-26T23:26:33,746 INFO [RS:0;c707e9ace675:41115 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C41115%2C1732663592583:(num 1732663593008) 2024-11-26T23:26:33,746 DEBUG [RS:0;c707e9ace675:41115 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:33,746 INFO [RS:0;c707e9ace675:41115 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:26:33,746 INFO [RS:0;c707e9ace675:41115 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:26:33,746 INFO [RS:0;c707e9ace675:41115 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T23:26:33,747 INFO [RS:0;c707e9ace675:41115 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:26:33,747 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:26:33,747 INFO [RS:0;c707e9ace675:41115 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41115 2024-11-26T23:26:33,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,41115,1732663592583 2024-11-26T23:26:33,749 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:26:33,749 INFO [RS:0;c707e9ace675:41115 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:26:33,752 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,41115,1732663592583] 2024-11-26T23:26:33,753 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,41115,1732663592583 already deleted, retry=false 2024-11-26T23:26:33,753 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c707e9ace675,41115,1732663592583 expired; onlineServers=0 2024-11-26T23:26:33,753 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c707e9ace675,44407,1732663592529' ***** 2024-11-26T23:26:33,754 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T23:26:33,754 INFO [M:0;c707e9ace675:44407 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:26:33,754 INFO [M:0;c707e9ace675:44407 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:26:33,754 DEBUG [M:0;c707e9ace675:44407 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T23:26:33,754 DEBUG [M:0;c707e9ace675:44407 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T23:26:33,754 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T23:26:33,754 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663592802 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663592802,5,FailOnTimeoutGroup] 2024-11-26T23:26:33,754 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663592802 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663592802,5,FailOnTimeoutGroup] 2024-11-26T23:26:33,754 INFO [M:0;c707e9ace675:44407 {}] hbase.ChoreService(370): Chore service for: master/c707e9ace675:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T23:26:33,754 INFO [M:0;c707e9ace675:44407 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:26:33,754 DEBUG [M:0;c707e9ace675:44407 {}] master.HMaster(1795): Stopping service threads 2024-11-26T23:26:33,755 INFO [M:0;c707e9ace675:44407 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T23:26:33,755 INFO [M:0;c707e9ace675:44407 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:26:33,755 INFO [M:0;c707e9ace675:44407 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T23:26:33,755 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T23:26:33,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T23:26:33,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:33,756 DEBUG [M:0;c707e9ace675:44407 {}] zookeeper.ZKUtil(347): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T23:26:33,756 WARN [M:0;c707e9ace675:44407 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T23:26:33,757 INFO [M:0;c707e9ace675:44407 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/.lastflushedseqids 2024-11-26T23:26:33,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741836_1012 (size=1) 2024-11-26T23:26:33,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741836_1012 (size=1) 2024-11-26T23:26:33,766 INFO [M:0;c707e9ace675:44407 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T23:26:33,766 INFO [M:0;c707e9ace675:44407 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T23:26:33,766 DEBUG [M:0;c707e9ace675:44407 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:26:33,766 INFO [M:0;c707e9ace675:44407 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:33,766 DEBUG [M:0;c707e9ace675:44407 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:33,766 DEBUG [M:0;c707e9ace675:44407 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:26:33,766 DEBUG [M:0;c707e9ace675:44407 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:33,767 INFO [M:0;c707e9ace675:44407 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-26T23:26:33,787 DEBUG [M:0;c707e9ace675:44407 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc3d955433404089948423c57b29bc9b is 82, key is hbase:meta,,1/info:regioninfo/1732663593465/Put/seqid=0 2024-11-26T23:26:33,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741837_1013 (size=5672) 2024-11-26T23:26:33,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741837_1013 (size=5672) 2024-11-26T23:26:33,793 INFO [M:0;c707e9ace675:44407 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc3d955433404089948423c57b29bc9b 2024-11-26T23:26:33,815 DEBUG [M:0;c707e9ace675:44407 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5ad80fff657c45279fcc86611d9168b5 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732663593489/Put/seqid=0 2024-11-26T23:26:33,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741838_1014 (size=5275) 2024-11-26T23:26:33,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741838_1014 (size=5275) 2024-11-26T23:26:33,821 INFO [M:0;c707e9ace675:44407 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5ad80fff657c45279fcc86611d9168b5 2024-11-26T23:26:33,842 DEBUG [M:0;c707e9ace675:44407 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/78588eb13ec845afb27ec499cfeca407 is 69, key is c707e9ace675,41115,1732663592583/rs:state/1732663592835/Put/seqid=0 2024-11-26T23:26:33,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741839_1015 (size=5156) 2024-11-26T23:26:33,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741839_1015 (size=5156) 2024-11-26T23:26:33,848 INFO [M:0;c707e9ace675:44407 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/78588eb13ec845afb27ec499cfeca407 2024-11-26T23:26:33,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:26:33,852 INFO [RS:0;c707e9ace675:41115 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:26:33,852 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41115-0x101569a30cf0001, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:26:33,852 INFO [RS:0;c707e9ace675:41115 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,41115,1732663592583; zookeeper connection closed. 2024-11-26T23:26:33,852 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@773a6b1b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@773a6b1b 2024-11-26T23:26:33,852 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T23:26:33,870 DEBUG [M:0;c707e9ace675:44407 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/53f7a1b74b7f4f9fa3912fb5a6f50f90 is 52, key is load_balancer_on/state:d/1732663593523/Put/seqid=0 2024-11-26T23:26:33,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741840_1016 (size=5056) 2024-11-26T23:26:33,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741840_1016 (size=5056) 2024-11-26T23:26:33,875 INFO [M:0;c707e9ace675:44407 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/53f7a1b74b7f4f9fa3912fb5a6f50f90 2024-11-26T23:26:33,882 DEBUG [M:0;c707e9ace675:44407 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc3d955433404089948423c57b29bc9b as hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc3d955433404089948423c57b29bc9b 2024-11-26T23:26:33,887 INFO [M:0;c707e9ace675:44407 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc3d955433404089948423c57b29bc9b, entries=8, sequenceid=29, filesize=5.5 K 2024-11-26T23:26:33,888 DEBUG [M:0;c707e9ace675:44407 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5ad80fff657c45279fcc86611d9168b5 as hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5ad80fff657c45279fcc86611d9168b5 2024-11-26T23:26:33,893 INFO [M:0;c707e9ace675:44407 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5ad80fff657c45279fcc86611d9168b5, entries=3, sequenceid=29, filesize=5.2 K 2024-11-26T23:26:33,894 DEBUG [M:0;c707e9ace675:44407 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/78588eb13ec845afb27ec499cfeca407 as hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/78588eb13ec845afb27ec499cfeca407 2024-11-26T23:26:33,900 INFO [M:0;c707e9ace675:44407 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/78588eb13ec845afb27ec499cfeca407, entries=1, sequenceid=29, filesize=5.0 K 2024-11-26T23:26:33,901 DEBUG [M:0;c707e9ace675:44407 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/53f7a1b74b7f4f9fa3912fb5a6f50f90 as hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/53f7a1b74b7f4f9fa3912fb5a6f50f90 2024-11-26T23:26:33,906 INFO [M:0;c707e9ace675:44407 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42807/user/jenkins/test-data/fda0a765-bb65-fc70-5b98-b7515b154a7f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/53f7a1b74b7f4f9fa3912fb5a6f50f90, entries=1, sequenceid=29, filesize=4.9 K 2024-11-26T23:26:33,907 INFO [M:0;c707e9ace675:44407 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=29, compaction requested=false 2024-11-26T23:26:33,911 INFO [M:0;c707e9ace675:44407 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:33,911 DEBUG [M:0;c707e9ace675:44407 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663593766Disabling compacts and flushes for region at 1732663593766Disabling writes for close at 1732663593766Obtaining lock to block concurrent updates at 1732663593767 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732663593767Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732663593767Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732663593768 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732663593768Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732663593786 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732663593786Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732663593799 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732663593814 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732663593814Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732663593827 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732663593842 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732663593842Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732663593854 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732663593869 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732663593869Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b63b0d2: reopening flushed file at 1732663593881 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2108ffcd: reopening flushed file at 1732663593887 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ee973cd: reopening flushed file at 1732663593893 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e67245a: reopening flushed file at 1732663593900 (+7 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 141ms, sequenceid=29, compaction requested=false at 1732663593907 (+7 ms)Writing region close event to WAL at 1732663593911 (+4 ms)Closed at 1732663593911 2024-11-26T23:26:33,912 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,912 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,912 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,912 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,912 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:33,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44775 is added to blk_1073741830_1006 (size=10311) 2024-11-26T23:26:33,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35705 is added to blk_1073741830_1006 (size=10311) 2024-11-26T23:26:33,915 INFO [M:0;c707e9ace675:44407 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T23:26:33,915 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:26:33,915 INFO [M:0;c707e9ace675:44407 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44407 2024-11-26T23:26:33,915 INFO [M:0;c707e9ace675:44407 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:26:34,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:26:34,018 INFO [M:0;c707e9ace675:44407 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:26:34,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44407-0x101569a30cf0000, quorum=127.0.0.1:63702, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:26:34,020 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@254b7745{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:34,021 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@86e294e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:26:34,021 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:26:34,021 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@64e5ce98{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:26:34,021 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@18f27499{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/hadoop.log.dir/,STOPPED} 2024-11-26T23:26:34,022 WARN [BP-1251012957-172.17.0.2-1732663591795 heartbeating to localhost/127.0.0.1:42807 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:26:34,022 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:26:34,023 WARN [BP-1251012957-172.17.0.2-1732663591795 heartbeating to localhost/127.0.0.1:42807 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1251012957-172.17.0.2-1732663591795 (Datanode Uuid 64d714f4-4608-4957-b402-a0653bb85756) service to localhost/127.0.0.1:42807 2024-11-26T23:26:34,023 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:26:34,023 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22/data/data3/current/BP-1251012957-172.17.0.2-1732663591795 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:34,023 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22/data/data4/current/BP-1251012957-172.17.0.2-1732663591795 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:34,024 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:26:34,026 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6a249094{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:34,026 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@35445cbd{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:26:34,026 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:26:34,026 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7247ee1d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:26:34,026 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@670e4080{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/hadoop.log.dir/,STOPPED} 2024-11-26T23:26:34,028 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:26:34,028 WARN [BP-1251012957-172.17.0.2-1732663591795 heartbeating to localhost/127.0.0.1:42807 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:26:34,028 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:26:34,028 WARN [BP-1251012957-172.17.0.2-1732663591795 heartbeating to localhost/127.0.0.1:42807 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1251012957-172.17.0.2-1732663591795 (Datanode Uuid 5b120619-752d-4b9e-b6e7-9daf2f10513c) service to localhost/127.0.0.1:42807 2024-11-26T23:26:34,028 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22/data/data1/current/BP-1251012957-172.17.0.2-1732663591795 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:34,029 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/cluster_0c510acd-02be-5451-f1ed-30eb3da7ed22/data/data2/current/BP-1251012957-172.17.0.2-1732663591795 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:34,029 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:26:34,035 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d195093{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:26:34,035 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@476d81{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:26:34,035 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:26:34,035 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@630e1a46{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:26:34,036 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1ae773f8{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/hadoop.log.dir/,STOPPED} 2024-11-26T23:26:34,042 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T23:26:34,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T23:26:34,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T23:26:34,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/hadoop.log.dir so I do NOT create it in target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217 2024-11-26T23:26:34,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/928c2f5e-0b12-afeb-199c-9c1f3be95206/hadoop.tmp.dir so I do NOT create it in target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217 2024-11-26T23:26:34,058 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8, deleteOnExit=true 2024-11-26T23:26:34,058 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T23:26:34,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/test.cache.data in system properties and HBase conf 2024-11-26T23:26:34,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T23:26:34,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir in system properties and HBase conf 2024-11-26T23:26:34,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T23:26:34,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T23:26:34,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T23:26:34,059 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T23:26:34,059 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:26:34,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:26:34,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T23:26:34,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:26:34,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T23:26:34,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T23:26:34,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:26:34,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:26:34,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T23:26:34,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/nfs.dump.dir in system properties and HBase conf 2024-11-26T23:26:34,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/java.io.tmpdir in system properties and HBase conf 2024-11-26T23:26:34,060 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:26:34,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T23:26:34,061 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T23:26:34,075 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:26:34,151 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:34,157 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:26:34,158 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:26:34,158 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:26:34,158 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:26:34,160 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:34,161 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@17c48ca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:26:34,161 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@35c95cb4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:26:34,285 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5519c514{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/java.io.tmpdir/jetty-localhost-46477-hadoop-hdfs-3_4_1-tests_jar-_-any-9299156008120990929/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:26:34,286 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@47f2ada2{HTTP/1.1, (http/1.1)}{localhost:46477} 2024-11-26T23:26:34,286 INFO [Time-limited test {}] server.Server(415): Started @103631ms 2024-11-26T23:26:34,300 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:26:34,371 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:34,375 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:26:34,376 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:26:34,377 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:26:34,377 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:26:34,377 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c3d2a60{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:26:34,378 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5917cb43{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:26:34,510 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1aa07d80{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/java.io.tmpdir/jetty-localhost-46085-hadoop-hdfs-3_4_1-tests_jar-_-any-13542853206137670785/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:34,510 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7c814f59{HTTP/1.1, (http/1.1)}{localhost:46085} 2024-11-26T23:26:34,510 INFO [Time-limited test {}] server.Server(415): Started @103855ms 2024-11-26T23:26:34,512 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:26:34,552 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:34,556 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:26:34,560 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:26:34,560 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:26:34,560 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:26:34,564 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@198c3788{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:26:34,564 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6471b09b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:26:34,635 WARN [Thread-656 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data1/current/BP-1916363482-172.17.0.2-1732663594093/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:34,636 WARN [Thread-657 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data2/current/BP-1916363482-172.17.0.2-1732663594093/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:34,682 WARN [Thread-635 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:26:34,687 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xefaa307a2983461b with lease ID 0x9f3d198b30680ff4: Processing first storage report for DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35 from datanode DatanodeRegistration(127.0.0.1:34663, datanodeUuid=44dee055-6f85-47c6-9cff-f0138c08f639, infoPort=35027, infoSecurePort=0, ipcPort=44355, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093) 2024-11-26T23:26:34,687 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xefaa307a2983461b with lease ID 0x9f3d198b30680ff4: from storage DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35 node DatanodeRegistration(127.0.0.1:34663, datanodeUuid=44dee055-6f85-47c6-9cff-f0138c08f639, infoPort=35027, infoSecurePort=0, ipcPort=44355, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:34,687 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xefaa307a2983461b with lease ID 0x9f3d198b30680ff4: Processing first storage report for DS-db067a98-e5eb-44b3-bdac-4ffaf89a39ff from datanode DatanodeRegistration(127.0.0.1:34663, datanodeUuid=44dee055-6f85-47c6-9cff-f0138c08f639, infoPort=35027, infoSecurePort=0, ipcPort=44355, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093) 2024-11-26T23:26:34,687 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xefaa307a2983461b with lease ID 0x9f3d198b30680ff4: from storage DS-db067a98-e5eb-44b3-bdac-4ffaf89a39ff node DatanodeRegistration(127.0.0.1:34663, datanodeUuid=44dee055-6f85-47c6-9cff-f0138c08f639, infoPort=35027, infoSecurePort=0, ipcPort=44355, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:34,704 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@55c8142a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/java.io.tmpdir/jetty-localhost-38793-hadoop-hdfs-3_4_1-tests_jar-_-any-3319723191095278064/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:34,705 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@65f2c48f{HTTP/1.1, (http/1.1)}{localhost:38793} 2024-11-26T23:26:34,705 INFO [Time-limited test {}] server.Server(415): Started @104050ms 2024-11-26T23:26:34,707 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:26:34,838 WARN [Thread-683 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data4/current/BP-1916363482-172.17.0.2-1732663594093/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:34,838 WARN [Thread-682 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data3/current/BP-1916363482-172.17.0.2-1732663594093/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:34,859 INFO [regionserver/c707e9ace675:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:26:34,867 WARN [Thread-671 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:26:34,871 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9af7884dadae516b with lease ID 0x9f3d198b30680ff5: Processing first storage report for DS-cacf0762-8e58-4aa7-98b9-88feb8cde337 from datanode DatanodeRegistration(127.0.0.1:40289, datanodeUuid=957de82d-0af5-4420-8b0f-1112e576e3dd, infoPort=40593, infoSecurePort=0, ipcPort=41455, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093) 2024-11-26T23:26:34,871 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9af7884dadae516b with lease ID 0x9f3d198b30680ff5: from storage DS-cacf0762-8e58-4aa7-98b9-88feb8cde337 node DatanodeRegistration(127.0.0.1:40289, datanodeUuid=957de82d-0af5-4420-8b0f-1112e576e3dd, infoPort=40593, infoSecurePort=0, ipcPort=41455, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:34,871 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9af7884dadae516b with lease ID 0x9f3d198b30680ff5: Processing first storage report for DS-0b538a44-15c7-42ec-a0e7-fd2bd11044d9 from datanode DatanodeRegistration(127.0.0.1:40289, datanodeUuid=957de82d-0af5-4420-8b0f-1112e576e3dd, infoPort=40593, infoSecurePort=0, ipcPort=41455, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093) 2024-11-26T23:26:34,871 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9af7884dadae516b with lease ID 0x9f3d198b30680ff5: from storage DS-0b538a44-15c7-42ec-a0e7-fd2bd11044d9 node DatanodeRegistration(127.0.0.1:40289, datanodeUuid=957de82d-0af5-4420-8b0f-1112e576e3dd, infoPort=40593, infoSecurePort=0, ipcPort=41455, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:34,975 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217 2024-11-26T23:26:34,979 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/zookeeper_0, clientPort=53009, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T23:26:34,980 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53009 2024-11-26T23:26:34,980 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:34,983 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:35,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34663 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:26:35,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40289 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:26:35,031 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad with version=8 2024-11-26T23:26:35,031 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/hbase-staging 2024-11-26T23:26:35,034 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:26:35,034 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:35,034 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:35,034 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:26:35,034 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:35,034 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:26:35,035 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T23:26:35,035 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:26:35,055 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33077 2024-11-26T23:26:35,057 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33077 connecting to ZooKeeper ensemble=127.0.0.1:53009 2024-11-26T23:26:35,077 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:330770x0, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:26:35,084 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33077-0x101569a3a720000 connected 2024-11-26T23:26:35,116 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:35,118 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:35,120 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:26:35,121 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad, hbase.cluster.distributed=false 2024-11-26T23:26:35,123 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:26:35,123 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33077 2024-11-26T23:26:35,123 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33077 2024-11-26T23:26:35,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33077 2024-11-26T23:26:35,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33077 2024-11-26T23:26:35,124 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33077 2024-11-26T23:26:35,141 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:26:35,141 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:35,141 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:35,141 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:26:35,141 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:35,141 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:26:35,141 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:26:35,141 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:26:35,142 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40529 2024-11-26T23:26:35,143 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40529 connecting to ZooKeeper ensemble=127.0.0.1:53009 2024-11-26T23:26:35,144 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:35,146 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:35,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:405290x0, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:26:35,151 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40529-0x101569a3a720001 connected 2024-11-26T23:26:35,151 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:26:35,151 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:26:35,152 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:26:35,153 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T23:26:35,154 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:26:35,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40529 2024-11-26T23:26:35,154 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40529 2024-11-26T23:26:35,155 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40529 2024-11-26T23:26:35,156 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40529 2024-11-26T23:26:35,157 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40529 2024-11-26T23:26:35,169 DEBUG [M:0;c707e9ace675:33077 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c707e9ace675:33077 2024-11-26T23:26:35,169 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c707e9ace675,33077,1732663595034 2024-11-26T23:26:35,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:26:35,171 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:26:35,172 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c707e9ace675,33077,1732663595034 2024-11-26T23:26:35,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T23:26:35,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,174 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T23:26:35,175 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c707e9ace675,33077,1732663595034 from backup master directory 2024-11-26T23:26:35,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c707e9ace675,33077,1732663595034 2024-11-26T23:26:35,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:26:35,177 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:26:35,177 WARN [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:26:35,177 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c707e9ace675,33077,1732663595034 2024-11-26T23:26:35,182 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/hbase.id] with ID: 732f4c4e-6002-41b6-b037-1d3368bbd0ea 2024-11-26T23:26:35,182 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/.tmp/hbase.id 2024-11-26T23:26:35,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34663 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:26:35,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40289 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:26:35,189 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/.tmp/hbase.id]:[hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/hbase.id] 2024-11-26T23:26:35,205 INFO [master/c707e9ace675:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:35,205 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T23:26:35,206 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-26T23:26:35,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,209 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40289 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:26:35,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34663 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:26:35,217 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:26:35,218 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T23:26:35,218 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:26:35,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34663 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:26:35,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40289 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:26:35,227 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store 2024-11-26T23:26:35,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34663 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:26:35,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40289 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:26:35,235 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:26:35,235 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:26:35,235 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:35,235 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:35,235 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:26:35,235 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:35,235 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:26:35,235 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663595235Disabling compacts and flushes for region at 1732663595235Disabling writes for close at 1732663595235Writing region close event to WAL at 1732663595235Closed at 1732663595235 2024-11-26T23:26:35,236 WARN [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/.initializing 2024-11-26T23:26:35,236 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034 2024-11-26T23:26:35,239 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C33077%2C1732663595034, suffix=, logDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034, archiveDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/oldWALs, maxLogs=10 2024-11-26T23:26:35,240 INFO [master/c707e9ace675:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C33077%2C1732663595034.1732663595240 2024-11-26T23:26:35,248 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034/c707e9ace675%2C33077%2C1732663595034.1732663595240 2024-11-26T23:26:35,250 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40593:40593),(127.0.0.1/127.0.0.1:35027:35027)] 2024-11-26T23:26:35,250 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:26:35,251 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:26:35,251 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,251 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,254 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T23:26:35,254 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,254 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:35,255 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,256 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T23:26:35,256 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,257 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:26:35,257 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T23:26:35,258 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:26:35,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,260 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T23:26:35,260 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,260 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:26:35,260 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,261 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,261 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,263 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,263 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,264 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:26:35,265 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:26:35,268 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:26:35,268 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=751773, jitterRate=-0.044072091579437256}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:26:35,270 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732663595251Initializing all the Stores at 1732663595252 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663595252Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663595252Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663595252Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663595252Cleaning up temporary data from old regions at 1732663595263 (+11 ms)Region opened successfully at 1732663595270 (+7 ms) 2024-11-26T23:26:35,270 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T23:26:35,274 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70720a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:26:35,275 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T23:26:35,275 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T23:26:35,275 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T23:26:35,275 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T23:26:35,276 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T23:26:35,276 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-26T23:26:35,276 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T23:26:35,278 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T23:26:35,279 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T23:26:35,280 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T23:26:35,281 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T23:26:35,281 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T23:26:35,282 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T23:26:35,283 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T23:26:35,284 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T23:26:35,286 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T23:26:35,286 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T23:26:35,287 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T23:26:35,289 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T23:26:35,290 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T23:26:35,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:26:35,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:26:35,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,292 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,293 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c707e9ace675,33077,1732663595034, sessionid=0x101569a3a720000, setting cluster-up flag (Was=false) 2024-11-26T23:26:35,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,302 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T23:26:35,303 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,33077,1732663595034 2024-11-26T23:26:35,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,313 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T23:26:35,314 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,33077,1732663595034 2024-11-26T23:26:35,315 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T23:26:35,317 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T23:26:35,318 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T23:26:35,318 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T23:26:35,318 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c707e9ace675,33077,1732663595034 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T23:26:35,320 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:26:35,320 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:26:35,320 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:26:35,320 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:26:35,320 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c707e9ace675:0, corePoolSize=10, maxPoolSize=10 2024-11-26T23:26:35,320 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,320 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:26:35,320 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,321 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732663625321 2024-11-26T23:26:35,321 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T23:26:35,321 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T23:26:35,321 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T23:26:35,321 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T23:26:35,321 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T23:26:35,321 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T23:26:35,322 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,322 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:26:35,322 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T23:26:35,323 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T23:26:35,323 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T23:26:35,323 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T23:26:35,323 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,323 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T23:26:35,324 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T23:26:35,324 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T23:26:35,325 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663595324,5,FailOnTimeoutGroup] 2024-11-26T23:26:35,325 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663595325,5,FailOnTimeoutGroup] 2024-11-26T23:26:35,325 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,325 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T23:26:35,325 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,325 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40289 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:26:35,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34663 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:26:35,334 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T23:26:35,334 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad 2024-11-26T23:26:35,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34663 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:26:35,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40289 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:26:35,342 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:26:35,343 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:26:35,345 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:26:35,345 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,345 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:35,346 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:26:35,347 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:26:35,347 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:35,347 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:26:35,349 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:26:35,349 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,349 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:35,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:26:35,351 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:26:35,351 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,351 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:35,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:26:35,352 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740 2024-11-26T23:26:35,353 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740 2024-11-26T23:26:35,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:26:35,354 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:26:35,355 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:26:35,356 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:26:35,358 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:26:35,359 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=825056, jitterRate=0.04911406338214874}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:26:35,359 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(746): ClusterId : 732f4c4e-6002-41b6-b037-1d3368bbd0ea 2024-11-26T23:26:35,359 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:26:35,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732663595342Initializing all the Stores at 1732663595343 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663595343Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663595343Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663595343Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663595343Cleaning up temporary data from old regions at 1732663595354 (+11 ms)Region opened successfully at 1732663595360 (+6 ms) 2024-11-26T23:26:35,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:26:35,360 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:26:35,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:26:35,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:26:35,360 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:26:35,361 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:26:35,361 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663595360Disabling compacts and flushes for region at 1732663595360Disabling writes for close at 1732663595360Writing region close event to WAL at 1732663595361 (+1 ms)Closed at 1732663595361 2024-11-26T23:26:35,362 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:26:35,362 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:26:35,362 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:26:35,362 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T23:26:35,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T23:26:35,364 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:26:35,364 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:26:35,365 DEBUG [RS:0;c707e9ace675:40529 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77728b0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:26:35,366 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T23:26:35,378 DEBUG [RS:0;c707e9ace675:40529 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c707e9ace675:40529 2024-11-26T23:26:35,379 INFO [RS:0;c707e9ace675:40529 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:26:35,379 INFO [RS:0;c707e9ace675:40529 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:26:35,379 DEBUG [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:26:35,379 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,33077,1732663595034 with port=40529, startcode=1732663595141 2024-11-26T23:26:35,380 DEBUG [RS:0;c707e9ace675:40529 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:26:35,417 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34807, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:26:35,418 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33077 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,40529,1732663595141 2024-11-26T23:26:35,418 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33077 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,40529,1732663595141 2024-11-26T23:26:35,421 DEBUG [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad 2024-11-26T23:26:35,421 DEBUG [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34607 2024-11-26T23:26:35,421 DEBUG [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:26:35,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:26:35,425 DEBUG [RS:0;c707e9ace675:40529 {}] zookeeper.ZKUtil(111): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,40529,1732663595141 2024-11-26T23:26:35,425 WARN [RS:0;c707e9ace675:40529 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:26:35,425 INFO [RS:0;c707e9ace675:40529 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:26:35,425 DEBUG [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141 2024-11-26T23:26:35,429 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,40529,1732663595141] 2024-11-26T23:26:35,435 INFO [RS:0;c707e9ace675:40529 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:26:35,440 INFO [RS:0;c707e9ace675:40529 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:26:35,440 INFO [RS:0;c707e9ace675:40529 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:26:35,440 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,441 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:26:35,442 INFO [RS:0;c707e9ace675:40529 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:26:35,442 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,442 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:26:35,443 DEBUG [RS:0;c707e9ace675:40529 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:26:35,444 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,444 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,444 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,444 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,444 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,444 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,40529,1732663595141-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:26:35,473 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:26:35,473 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,40529,1732663595141-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,473 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,473 INFO [RS:0;c707e9ace675:40529 {}] regionserver.Replication(171): c707e9ace675,40529,1732663595141 started 2024-11-26T23:26:35,496 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,497 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,40529,1732663595141, RpcServer on c707e9ace675/172.17.0.2:40529, sessionid=0x101569a3a720001 2024-11-26T23:26:35,497 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:26:35,497 DEBUG [RS:0;c707e9ace675:40529 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,40529,1732663595141 2024-11-26T23:26:35,497 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,40529,1732663595141' 2024-11-26T23:26:35,497 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:26:35,497 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:26:35,498 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:26:35,498 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:26:35,498 DEBUG [RS:0;c707e9ace675:40529 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,40529,1732663595141 2024-11-26T23:26:35,498 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,40529,1732663595141' 2024-11-26T23:26:35,498 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:26:35,498 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:26:35,499 DEBUG [RS:0;c707e9ace675:40529 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:26:35,499 INFO [RS:0;c707e9ace675:40529 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:26:35,499 INFO [RS:0;c707e9ace675:40529 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:26:35,516 WARN [c707e9ace675:33077 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T23:26:35,601 INFO [RS:0;c707e9ace675:40529 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C40529%2C1732663595141, suffix=, logDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141, archiveDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs, maxLogs=32 2024-11-26T23:26:35,602 INFO [RS:0;c707e9ace675:40529 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C40529%2C1732663595141.1732663595602 2024-11-26T23:26:35,609 INFO [RS:0;c707e9ace675:40529 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 2024-11-26T23:26:35,610 DEBUG [RS:0;c707e9ace675:40529 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:40593:40593),(127.0.0.1/127.0.0.1:35027:35027)] 2024-11-26T23:26:35,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:26:35,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T23:26:35,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testSlowSyncLogRolling 2024-11-26T23:26:35,766 DEBUG [c707e9ace675:33077 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T23:26:35,767 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c707e9ace675,40529,1732663595141 2024-11-26T23:26:35,768 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,40529,1732663595141, state=OPENING 2024-11-26T23:26:35,770 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T23:26:35,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:26:35,772 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:26:35,772 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:26:35,772 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,40529,1732663595141}] 2024-11-26T23:26:35,772 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:26:35,925 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:26:35,927 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36641, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:26:35,931 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T23:26:35,931 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:26:35,933 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C40529%2C1732663595141.meta, suffix=.meta, logDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141, archiveDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs, maxLogs=32 2024-11-26T23:26:35,934 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta 2024-11-26T23:26:35,938 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta 2024-11-26T23:26:35,939 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35027:35027),(127.0.0.1/127.0.0.1:40593:40593)] 2024-11-26T23:26:35,940 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:26:35,940 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T23:26:35,940 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T23:26:35,940 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T23:26:35,941 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T23:26:35,941 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:26:35,941 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T23:26:35,941 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T23:26:35,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:26:35,943 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:26:35,943 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,943 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:35,944 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:26:35,944 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:26:35,944 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:35,945 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:26:35,946 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:26:35,946 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:35,946 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:26:35,947 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:26:35,947 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:35,947 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:26:35,948 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:26:35,948 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740 2024-11-26T23:26:35,949 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740 2024-11-26T23:26:35,950 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:26:35,951 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:26:35,951 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:26:35,952 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:26:35,953 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=861697, jitterRate=0.09570562839508057}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:26:35,953 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T23:26:35,954 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732663595941Writing region info on filesystem at 1732663595941Initializing all the Stores at 1732663595942 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663595942Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663595942Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663595942Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663595942Cleaning up temporary data from old regions at 1732663595951 (+9 ms)Running coprocessor post-open hooks at 1732663595953 (+2 ms)Region opened successfully at 1732663595954 (+1 ms) 2024-11-26T23:26:35,955 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732663595925 2024-11-26T23:26:35,958 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T23:26:35,958 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T23:26:35,959 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,40529,1732663595141 2024-11-26T23:26:35,960 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,40529,1732663595141, state=OPEN 2024-11-26T23:26:35,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:26:35,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:26:35,965 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c707e9ace675,40529,1732663595141 2024-11-26T23:26:35,965 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:26:35,965 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:26:35,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T23:26:35,968 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,40529,1732663595141 in 193 msec 2024-11-26T23:26:35,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T23:26:35,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 605 msec 2024-11-26T23:26:35,972 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:26:35,972 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T23:26:35,973 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:26:35,973 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,40529,1732663595141, seqNum=-1] 2024-11-26T23:26:35,974 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:26:35,975 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43889, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:26:35,981 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 663 msec 2024-11-26T23:26:35,981 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732663595981, completionTime=-1 2024-11-26T23:26:35,981 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T23:26:35,981 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T23:26:35,983 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T23:26:35,983 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732663655983 2024-11-26T23:26:35,983 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732663715983 2024-11-26T23:26:35,983 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 2 msec 2024-11-26T23:26:35,984 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,33077,1732663595034-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,984 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,33077,1732663595034-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,984 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,33077,1732663595034-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,984 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c707e9ace675:33077, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,984 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,984 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:35,986 DEBUG [master/c707e9ace675:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T23:26:35,988 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.811sec 2024-11-26T23:26:35,988 INFO [master/c707e9ace675:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T23:26:35,988 INFO [master/c707e9ace675:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T23:26:35,988 INFO [master/c707e9ace675:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T23:26:35,988 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T23:26:35,988 INFO [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T23:26:35,988 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,33077,1732663595034-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:26:35,988 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,33077,1732663595034-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T23:26:35,990 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T23:26:35,991 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T23:26:35,991 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,33077,1732663595034-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:36,062 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10c7ae56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:26:36,062 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,33077,-1 for getting cluster id 2024-11-26T23:26:36,062 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:26:36,064 DEBUG [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '732f4c4e-6002-41b6-b037-1d3368bbd0ea' 2024-11-26T23:26:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:26:36,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "732f4c4e-6002-41b6-b037-1d3368bbd0ea" 2024-11-26T23:26:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b0ca97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:26:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,33077,-1] 2024-11-26T23:26:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:26:36,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:26:36,067 INFO [HMaster-EventLoopGroup-7-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:26:36,068 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@272348fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:26:36,068 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:26:36,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,40529,1732663595141, seqNum=-1] 2024-11-26T23:26:36,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:26:36,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-8-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:26:36,073 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c707e9ace675,33077,1732663595034 2024-11-26T23:26:36,073 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:36,077 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-26T23:26:36,093 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:26:36,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:36,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:36,093 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:26:36,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:26:36,093 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:26:36,093 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:26:36,093 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:26:36,094 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37097 2024-11-26T23:26:36,095 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:37097 connecting to ZooKeeper ensemble=127.0.0.1:53009 2024-11-26T23:26:36,096 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:36,097 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:26:36,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:370970x0, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:26:36,103 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:370970x0, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on existing znode=/hbase/running 2024-11-26T23:26:36,103 DEBUG [pool-381-thread-1 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: INIT 2024-11-26T23:26:36,103 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:37097-0x101569a3a720002 connected 2024-11-26T23:26:36,103 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:26:36,104 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:26:36,105 DEBUG [Time-limited test {}] zookeeper.ZKUtil(111): regionserver:37097-0x101569a3a720002, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T23:26:36,106 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:37097-0x101569a3a720002, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:26:36,108 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37097 2024-11-26T23:26:36,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37097 2024-11-26T23:26:36,109 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37097 2024-11-26T23:26:36,112 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37097 2024-11-26T23:26:36,113 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37097 2024-11-26T23:26:36,114 INFO [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer(746): ClusterId : 732f4c4e-6002-41b6-b037-1d3368bbd0ea 2024-11-26T23:26:36,114 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:26:36,116 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:26:36,116 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:26:36,119 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:26:36,120 DEBUG [RS:1;c707e9ace675:37097 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f030315, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:26:36,131 DEBUG [RS:1;c707e9ace675:37097 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c707e9ace675:37097 2024-11-26T23:26:36,132 INFO [RS:1;c707e9ace675:37097 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:26:36,132 INFO [RS:1;c707e9ace675:37097 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:26:36,132 DEBUG [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:26:36,132 INFO [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,33077,1732663595034 with port=37097, startcode=1732663596093 2024-11-26T23:26:36,133 DEBUG [RS:1;c707e9ace675:37097 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:26:36,134 INFO [HMaster-EventLoopGroup-7-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49945, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.3 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:26:36,135 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33077 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,37097,1732663596093 2024-11-26T23:26:36,135 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33077 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,37097,1732663596093 2024-11-26T23:26:36,136 DEBUG [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad 2024-11-26T23:26:36,136 DEBUG [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34607 2024-11-26T23:26:36,137 DEBUG [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:26:36,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:26:36,139 DEBUG [RS:1;c707e9ace675:37097 {}] zookeeper.ZKUtil(111): regionserver:37097-0x101569a3a720002, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,37097,1732663596093 2024-11-26T23:26:36,139 WARN [RS:1;c707e9ace675:37097 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:26:36,139 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,37097,1732663596093] 2024-11-26T23:26:36,139 INFO [RS:1;c707e9ace675:37097 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:26:36,139 DEBUG [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093 2024-11-26T23:26:36,143 INFO [RS:1;c707e9ace675:37097 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:26:36,145 INFO [RS:1;c707e9ace675:37097 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:26:36,146 INFO [RS:1;c707e9ace675:37097 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:26:36,146 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:36,146 INFO [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:26:36,147 INFO [RS:1;c707e9ace675:37097 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:26:36,147 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:36,147 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:36,147 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:36,147 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:36,147 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:36,147 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:36,147 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:26:36,147 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:36,148 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:36,148 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:36,148 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:36,148 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:36,148 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:26:36,148 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:26:36,148 DEBUG [RS:1;c707e9ace675:37097 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:26:36,148 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:36,148 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:36,148 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:36,148 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:36,148 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:36,148 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,37097,1732663596093-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:26:36,170 INFO [RS:1;c707e9ace675:37097 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:26:36,170 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,37097,1732663596093-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:36,170 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:36,170 INFO [RS:1;c707e9ace675:37097 {}] regionserver.Replication(171): c707e9ace675,37097,1732663596093 started 2024-11-26T23:26:36,184 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:26:36,184 INFO [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,37097,1732663596093, RpcServer on c707e9ace675/172.17.0.2:37097, sessionid=0x101569a3a720002 2024-11-26T23:26:36,184 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:26:36,184 DEBUG [RS:1;c707e9ace675:37097 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,37097,1732663596093 2024-11-26T23:26:36,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(2882): Started new server=Thread[RS:1;c707e9ace675:37097,5,FailOnTimeoutGroup] 2024-11-26T23:26:36,184 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,37097,1732663596093' 2024-11-26T23:26:36,184 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:26:36,185 INFO [Time-limited test {}] wal.TestLogRolling(207): Replication=2 2024-11-26T23:26:36,185 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:26:36,185 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:26:36,185 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:26:36,185 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:26:36,186 DEBUG [RS:1;c707e9ace675:37097 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,37097,1732663596093 2024-11-26T23:26:36,186 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,37097,1732663596093' 2024-11-26T23:26:36,186 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:26:36,186 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:26:36,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,33077,1732663595034 2024-11-26T23:26:36,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@72aff3bb 2024-11-26T23:26:36,186 DEBUG [RS:1;c707e9ace675:37097 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:26:36,186 INFO [RS:1;c707e9ace675:37097 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:26:36,187 INFO [RS:1;c707e9ace675:37097 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:26:36,187 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:26:36,188 INFO [HMaster-EventLoopGroup-7-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:26:36,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33077 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-26T23:26:36,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33077 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-26T23:26:36,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33077 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnDatanodeDeath', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:26:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33077 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath 2024-11-26T23:26:36,192 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:26:36,192 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:36,192 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33077 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnDatanodeDeath" procId is: 4 2024-11-26T23:26:36,193 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:26:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33077 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:26:36,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34663 is added to blk_1073741835_1011 (size=393) 2024-11-26T23:26:36,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40289 is added to blk_1073741835_1011 (size=393) 2024-11-26T23:26:36,207 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6644eb6ad870b0b346267b71ec1a8f5b, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnDatanodeDeath', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad 2024-11-26T23:26:36,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34663 is added to blk_1073741836_1012 (size=76) 2024-11-26T23:26:36,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40289 is added to blk_1073741836_1012 (size=76) 2024-11-26T23:26:36,216 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:26:36,216 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1722): Closing 6644eb6ad870b0b346267b71ec1a8f5b, disabling compactions & flushes 2024-11-26T23:26:36,216 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:26:36,216 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:26:36,217 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. after waiting 0 ms 2024-11-26T23:26:36,217 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:26:36,217 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:26:36,217 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnDatanodeDeath-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6644eb6ad870b0b346267b71ec1a8f5b: Waiting for close lock at 1732663596216Disabling compacts and flushes for region at 1732663596216Disabling writes for close at 1732663596217 (+1 ms)Writing region close event to WAL at 1732663596217Closed at 1732663596217 2024-11-26T23:26:36,218 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:26:36,219 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.","families":{"info":[{"qualifier":"regioninfo","vlen":75,"tag":[],"timestamp":"1732663596218"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732663596218"}]},"ts":"1732663596218"} 2024-11-26T23:26:36,221 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:26:36,222 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:26:36,223 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663596222"}]},"ts":"1732663596222"} 2024-11-26T23:26:36,224 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLING in hbase:meta 2024-11-26T23:26:36,225 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=6644eb6ad870b0b346267b71ec1a8f5b, ASSIGN}] 2024-11-26T23:26:36,226 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=6644eb6ad870b0b346267b71ec1a8f5b, ASSIGN 2024-11-26T23:26:36,227 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=6644eb6ad870b0b346267b71ec1a8f5b, ASSIGN; state=OFFLINE, location=c707e9ace675,40529,1732663595141; forceNewPlan=false, retain=false 2024-11-26T23:26:36,233 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:26:36,236 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:26:36,289 INFO [RS:1;c707e9ace675:37097 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C37097%2C1732663596093, suffix=, logDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093, archiveDir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs, maxLogs=32 2024-11-26T23:26:36,289 INFO [RS:1;c707e9ace675:37097 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C37097%2C1732663596093.1732663596289 2024-11-26T23:26:36,295 INFO [RS:1;c707e9ace675:37097 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 2024-11-26T23:26:36,296 DEBUG [RS:1;c707e9ace675:37097 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:35027:35027),(127.0.0.1/127.0.0.1:40593:40593)] 2024-11-26T23:26:36,378 INFO [c707e9ace675:33077 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-26T23:26:36,379 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6644eb6ad870b0b346267b71ec1a8f5b, regionState=OPENING, regionLocation=c707e9ace675,40529,1732663595141 2024-11-26T23:26:36,381 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=6644eb6ad870b0b346267b71ec1a8f5b, ASSIGN because future has completed 2024-11-26T23:26:36,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6644eb6ad870b0b346267b71ec1a8f5b, server=c707e9ace675,40529,1732663595141}] 2024-11-26T23:26:36,539 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:26:36,539 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6644eb6ad870b0b346267b71ec1a8f5b, NAME => 'TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:26:36,540 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnDatanodeDeath 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:36,540 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:26:36,540 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:36,540 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:36,542 INFO [StoreOpener-6644eb6ad870b0b346267b71ec1a8f5b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:36,543 INFO [StoreOpener-6644eb6ad870b0b346267b71ec1a8f5b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6644eb6ad870b0b346267b71ec1a8f5b columnFamilyName info 2024-11-26T23:26:36,543 DEBUG [StoreOpener-6644eb6ad870b0b346267b71ec1a8f5b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:26:36,543 INFO [StoreOpener-6644eb6ad870b0b346267b71ec1a8f5b-1 {}] regionserver.HStore(327): Store=6644eb6ad870b0b346267b71ec1a8f5b/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:26:36,544 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:36,544 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:36,545 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:36,545 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:36,545 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:36,546 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:36,548 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:26:36,549 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6644eb6ad870b0b346267b71ec1a8f5b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=698884, jitterRate=-0.11132392287254333}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:26:36,549 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:36,550 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6644eb6ad870b0b346267b71ec1a8f5b: Running coprocessor pre-open hook at 1732663596540Writing region info on filesystem at 1732663596540Initializing all the Stores at 1732663596541 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663596541Cleaning up temporary data from old regions at 1732663596545 (+4 ms)Running coprocessor post-open hooks at 1732663596549 (+4 ms)Region opened successfully at 1732663596549 2024-11-26T23:26:36,551 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b., pid=6, masterSystemTime=1732663596535 2024-11-26T23:26:36,553 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:26:36,553 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:26:36,554 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6644eb6ad870b0b346267b71ec1a8f5b, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,40529,1732663595141 2024-11-26T23:26:36,556 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-8-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6644eb6ad870b0b346267b71ec1a8f5b, server=c707e9ace675,40529,1732663595141 because future has completed 2024-11-26T23:26:36,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T23:26:36,560 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6644eb6ad870b0b346267b71ec1a8f5b, server=c707e9ace675,40529,1732663595141 in 176 msec 2024-11-26T23:26:36,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T23:26:36,563 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnDatanodeDeath, region=6644eb6ad870b0b346267b71ec1a8f5b, ASSIGN in 335 msec 2024-11-26T23:26:36,564 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:26:36,564 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnDatanodeDeath","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663596564"}]},"ts":"1732663596564"} 2024-11-26T23:26:36,566 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnDatanodeDeath, state=ENABLED in hbase:meta 2024-11-26T23:26:36,567 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:26:36,569 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnDatanodeDeath in 378 msec 2024-11-26T23:26:36,746 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-26T23:26:36,747 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:26:36,763 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:26:36,765 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:26:36,766 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:26:41,436 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnDatanodeDeath' 2024-11-26T23:26:42,042 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-26T23:26:42,044 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:26:42,059 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:26:42,064 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:26:42,065 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:26:45,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T23:26:45,734 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T23:26:45,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-26T23:26:45,735 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath Metrics about Tables on a single HBase RegionServer 2024-11-26T23:26:45,736 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:26:45,736 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T23:26:46,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33077 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:26:46,218 INFO [RPCClient-NioEventLoopGroup-4-11 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnDatanodeDeath completed 2024-11-26T23:26:46,218 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnDatanodeDeath,, stopping at row=TestLogRolling-testLogRollOnDatanodeDeath ,, for max=2147483647 with caching=100 2024-11-26T23:26:46,221 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnDatanodeDeath 2024-11-26T23:26:46,221 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:26:46,234 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:46,237 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:26:46,238 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:26:46,238 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:26:46,238 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:26:46,239 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@219c70cc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:26:46,239 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75255721{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:26:46,352 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@22d0350b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/java.io.tmpdir/jetty-localhost-34709-hadoop-hdfs-3_4_1-tests_jar-_-any-5507528621522862768/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:46,353 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3cc6081e{HTTP/1.1, (http/1.1)}{localhost:34709} 2024-11-26T23:26:46,353 INFO [Time-limited test {}] server.Server(415): Started @115698ms 2024-11-26T23:26:46,354 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:26:46,389 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:46,393 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:26:46,394 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:26:46,394 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:26:46,394 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:26:46,395 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fb99827{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:26:46,395 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7fbc343d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:26:46,453 WARN [Thread-828 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data5/current/BP-1916363482-172.17.0.2-1732663594093/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:46,453 WARN [Thread-829 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data6/current/BP-1916363482-172.17.0.2-1732663594093/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:46,479 WARN [Thread-808 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:26:46,481 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb2110362b558f6e6 with lease ID 0x9f3d198b30680ff6: Processing first storage report for DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4 from datanode DatanodeRegistration(127.0.0.1:39209, datanodeUuid=75864b72-31eb-4f65-97d9-6a407e9e7835, infoPort=46769, infoSecurePort=0, ipcPort=44173, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093) 2024-11-26T23:26:46,481 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb2110362b558f6e6 with lease ID 0x9f3d198b30680ff6: from storage DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4 node DatanodeRegistration(127.0.0.1:39209, datanodeUuid=75864b72-31eb-4f65-97d9-6a407e9e7835, infoPort=46769, infoSecurePort=0, ipcPort=44173, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:46,481 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb2110362b558f6e6 with lease ID 0x9f3d198b30680ff6: Processing first storage report for DS-e015c681-257e-4e05-af9a-98bce41777a0 from datanode DatanodeRegistration(127.0.0.1:39209, datanodeUuid=75864b72-31eb-4f65-97d9-6a407e9e7835, infoPort=46769, infoSecurePort=0, ipcPort=44173, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093) 2024-11-26T23:26:46,481 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb2110362b558f6e6 with lease ID 0x9f3d198b30680ff6: from storage DS-e015c681-257e-4e05-af9a-98bce41777a0 node DatanodeRegistration(127.0.0.1:39209, datanodeUuid=75864b72-31eb-4f65-97d9-6a407e9e7835, infoPort=46769, infoSecurePort=0, ipcPort=44173, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:46,523 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3df1987c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/java.io.tmpdir/jetty-localhost-32937-hadoop-hdfs-3_4_1-tests_jar-_-any-5528617116738153474/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:46,524 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@71bfb2ba{HTTP/1.1, (http/1.1)}{localhost:32937} 2024-11-26T23:26:46,524 INFO [Time-limited test {}] server.Server(415): Started @115869ms 2024-11-26T23:26:46,526 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:26:46,571 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:46,574 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:26:46,574 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:26:46,574 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:26:46,574 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:26:46,575 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41559526{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:26:46,575 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@495a6aea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:26:46,636 WARN [Thread-864 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8/current/BP-1916363482-172.17.0.2-1732663594093/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:46,636 WARN [Thread-863 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7/current/BP-1916363482-172.17.0.2-1732663594093/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:46,657 WARN [Thread-843 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:26:46,659 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf70465809bb3d72c with lease ID 0x9f3d198b30680ff7: Processing first storage report for DS-649476ef-55b2-41c2-b8a9-b56007473276 from datanode DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093) 2024-11-26T23:26:46,659 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf70465809bb3d72c with lease ID 0x9f3d198b30680ff7: from storage DS-649476ef-55b2-41c2-b8a9-b56007473276 node DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:46,660 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf70465809bb3d72c with lease ID 0x9f3d198b30680ff7: Processing first storage report for DS-6f1de9df-023d-4a4d-81f9-eb03669d5eb8 from datanode DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093) 2024-11-26T23:26:46,660 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf70465809bb3d72c with lease ID 0x9f3d198b30680ff7: from storage DS-6f1de9df-023d-4a4d-81f9-eb03669d5eb8 node DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-26T23:26:46,692 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@f0760d{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/java.io.tmpdir/jetty-localhost-43195-hadoop-hdfs-3_4_1-tests_jar-_-any-15358594298604272634/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:46,693 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@21f536ea{HTTP/1.1, (http/1.1)}{localhost:43195} 2024-11-26T23:26:46,693 INFO [Time-limited test {}] server.Server(415): Started @116038ms 2024-11-26T23:26:46,694 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:26:46,801 WARN [Thread-889 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data9/current/BP-1916363482-172.17.0.2-1732663594093/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:46,801 WARN [Thread-890 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data10/current/BP-1916363482-172.17.0.2-1732663594093/current, will proceed with Du for space computation calculation, 2024-11-26T23:26:46,819 WARN [Thread-878 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:26:46,821 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x94aa22ba683d286e with lease ID 0x9f3d198b30680ff8: Processing first storage report for DS-550c15a6-fa37-4e21-8f11-28a425daf14b from datanode DatanodeRegistration(127.0.0.1:44117, datanodeUuid=6496249a-f74a-4959-91cb-cd9c92e098c4, infoPort=38187, infoSecurePort=0, ipcPort=34847, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093) 2024-11-26T23:26:46,821 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x94aa22ba683d286e with lease ID 0x9f3d198b30680ff8: from storage DS-550c15a6-fa37-4e21-8f11-28a425daf14b node DatanodeRegistration(127.0.0.1:44117, datanodeUuid=6496249a-f74a-4959-91cb-cd9c92e098c4, infoPort=38187, infoSecurePort=0, ipcPort=34847, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:46,821 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x94aa22ba683d286e with lease ID 0x9f3d198b30680ff8: Processing first storage report for DS-dbf4b040-b6cf-4b94-8a7a-d874cbc3a4b8 from datanode DatanodeRegistration(127.0.0.1:44117, datanodeUuid=6496249a-f74a-4959-91cb-cd9c92e098c4, infoPort=38187, infoSecurePort=0, ipcPort=34847, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093) 2024-11-26T23:26:46,821 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x94aa22ba683d286e with lease ID 0x9f3d198b30680ff8: from storage DS-dbf4b040-b6cf-4b94-8a7a-d874cbc3a4b8 node DatanodeRegistration(127.0.0.1:44117, datanodeUuid=6496249a-f74a-4959-91cb-cd9c92e098c4, infoPort=38187, infoSecurePort=0, ipcPort=34847, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:46,915 WARN [ResponseProcessor for block BP-1916363482-172.17.0.2-1732663594093:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1916363482-172.17.0.2-1732663594093:blk_1073741830_1006 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,916 WARN [ResponseProcessor for block BP-1916363482-172.17.0.2-1732663594093:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1916363482-172.17.0.2-1732663594093:blk_1073741833_1009 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,916 WARN [ResponseProcessor for block BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010 java.io.IOException: Bad response ERROR for BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010 from datanode DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,916 WARN [DataStreamer for file /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 block BP-1916363482-172.17.0.2-1732663594093:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK], DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:46,916 WARN [DataStreamer for file /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta block BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK], DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:46,916 WARN [DataStreamer for file /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034/c707e9ace675%2C33077%2C1732663595034.1732663595240 block BP-1916363482-172.17.0.2-1732663594093:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK], DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:46,917 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:60252 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:40289:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60252 dst: /127.0.0.1:40289 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:46,917 WARN [PacketResponder: BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:40289] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:46,917 WARN [ResponseProcessor for block BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013 java.io.IOException: Bad response ERROR for BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013 from datanode DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,918 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:39992 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:34663:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39992 dst: /127.0.0.1:34663 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:46,918 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@55c8142a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:46,918 WARN [DataStreamer for file /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 block BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK], DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:46,918 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@65f2c48f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:26:46,918 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2114532505_22 at /127.0.0.1:39980 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:34663:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39980 dst: /127.0.0.1:34663 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:46,918 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_2114532505_22 at /127.0.0.1:60232 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:40289:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60232 dst: /127.0.0.1:40289 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:46,919 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:26:46,919 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6471b09b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:26:46,919 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:40002 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:34663:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40002 dst: /127.0.0.1:34663 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:46,919 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@198c3788{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,STOPPED} 2024-11-26T23:26:46,919 WARN [PacketResponder: BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:40289] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:46,919 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:60258 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:40289:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60258 dst: /127.0.0.1:40289 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:46,919 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1005798636_22 at /127.0.0.1:40012 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:34663:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40012 dst: /127.0.0.1:34663 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:46,920 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:26:46,920 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1916363482-172.17.0.2-1732663594093 (Datanode Uuid 957de82d-0af5-4420-8b0f-1112e576e3dd) service to localhost/127.0.0.1:34607 2024-11-26T23:26:46,920 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1005798636_22 at /127.0.0.1:60292 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013] {}] datanode.DataXceiver(331): 127.0.0.1:40289:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:60292 dst: /127.0.0.1:40289 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:46,921 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data3/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:46,921 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data4/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:46,921 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:26:46,922 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:26:46,922 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:26:46,924 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1aa07d80{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:46,925 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7c814f59{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:26:46,925 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:26:46,925 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5917cb43{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:26:46,926 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c3d2a60{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,STOPPED} 2024-11-26T23:26:46,926 WARN [DataStreamer for file /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034/c707e9ace675%2C33077%2C1732663595034.1732663595240 block BP-1916363482-172.17.0.2-1732663594093:blk_1073741830_1006 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741830_1006 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,926 WARN [DataStreamer for file /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 block BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741837_1013 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,926 WARN [DataStreamer for file /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 block BP-1916363482-172.17.0.2-1732663594093:blk_1073741833_1009 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741833_1009 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,926 WARN [DataStreamer for file /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta block BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741834_1010 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1666) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,927 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:26:46,927 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1916363482-172.17.0.2-1732663594093 (Datanode Uuid 44dee055-6f85-47c6-9cff-f0138c08f639) service to localhost/127.0.0.1:34607 2024-11-26T23:26:46,927 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:26:46,927 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:26:46,928 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data1/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:46,928 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data2/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:46,928 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:26:46,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnDatanodeDeath', row='row0002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b., hostname=c707e9ace675,40529,1732663595141, seqNum=2] 2024-11-26T23:26:46,934 ERROR [FSHLog-0-hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad-prefix:c707e9ace675,40529,1732663595141 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,934 WARN [FSHLog-0-hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad-prefix:c707e9ace675,40529,1732663595141 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,935 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,935 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C40529%2C1732663595141:(num 1732663595602) roll requested 2024-11-26T23:26:46,935 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C40529%2C1732663595141.1732663606935 2024-11-26T23:26:46,941 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:46,941 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:46,941 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:46,941 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:46,941 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:46,941 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 with entries=1, filesize=455 B; new WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663606935 2024-11-26T23:26:46,942 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,942 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:46,942 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:38187:38187),(127.0.0.1/127.0.0.1:46769:46769)] 2024-11-26T23:26:46,942 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 is not closed yet, will try archiving it next time 2024-11-26T23:26:46,943 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-26T23:26:46,943 DEBUG [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-26T23:26:46,943 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 2024-11-26T23:26:46,946 WARN [IPC Server handler 4 on default port 34607 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 has not been closed. Lease recovery is in progress. RecoveryId = 1019 for block blk_1073741833_1009 2024-11-26T23:26:46,950 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 after 5ms 2024-11-26T23:26:47,219 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:48,149 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:48,942 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:48,944 INFO [Time-limited test {}] wal.TestLogRolling(261): log.getCurrentFileName(): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663606935 2024-11-26T23:26:48,944 WARN [ResponseProcessor for block BP-1916363482-172.17.0.2-1732663594093:blk_1073741838_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1916363482-172.17.0.2-1732663594093:blk_1073741838_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:48,945 WARN [DataStreamer for file /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663606935 block BP-1916363482-172.17.0.2-1732663594093:blk_1073741838_1018 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741838_1018 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK], DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]) is bad. 2024-11-26T23:26:48,945 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:57234 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:44117:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57234 dst: /127.0.0.1:44117 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:48,945 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:56004 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741838_1018] {}] datanode.DataXceiver(331): 127.0.0.1:39209:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56004 dst: /127.0.0.1:39209 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:48,947 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@f0760d{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:48,947 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@21f536ea{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:26:48,948 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:26:48,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@495a6aea{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:26:48,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41559526{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,STOPPED} 2024-11-26T23:26:48,952 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:26:48,952 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:26:48,952 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:26:48,952 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1916363482-172.17.0.2-1732663594093 (Datanode Uuid 6496249a-f74a-4959-91cb-cd9c92e098c4) service to localhost/127.0.0.1:34607 2024-11-26T23:26:48,952 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data9/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:48,953 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data10/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:48,953 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:26:49,219 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:50,149 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:50,943 WARN [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]] 2024-11-26T23:26:50,943 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:50,944 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C40529%2C1732663595141:(num 1732663606935) roll requested 2024-11-26T23:26:50,944 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C40529%2C1732663595141.1732663610944 2024-11-26T23:26:50,950 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:50,950 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:50,951 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:50,951 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:50,951 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:50,951 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 after 4008ms 2024-11-26T23:26:50,951 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663606935 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663610944 2024-11-26T23:26:50,952 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:46769:46769),(127.0.0.1/127.0.0.1:34309:34309)] 2024-11-26T23:26:50,952 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 is not closed yet, will try archiving it next time 2024-11-26T23:26:50,952 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663606935 is not closed yet, will try archiving it next time 2024-11-26T23:26:50,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39209 is added to blk_1073741838_1020 (size=2431) 2024-11-26T23:26:50,957 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [10,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T23:26:51,220 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:51,354 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 is not closed yet, will try archiving it next time 2024-11-26T23:26:52,150 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:52,493 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6ac0d200[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:39209, datanodeUuid=75864b72-31eb-4f65-97d9-6a407e9e7835, infoPort=46769, infoSecurePort=0, ipcPort=44173, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741838_1020 to 127.0.0.1:44117 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:52,952 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:52,961 WARN [ResponseProcessor for block BP-1916363482-172.17.0.2-1732663594093:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1916363482-172.17.0.2-1732663594093:blk_1073741839_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:52,961 WARN [DataStreamer for file /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663610944 block BP-1916363482-172.17.0.2-1732663594093:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:26:52,961 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:45568 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:39209:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45568 dst: /127.0.0.1:39209 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:52,962 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47168 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47168 dst: /127.0.0.1:44973 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:52,963 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@22d0350b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:52,963 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3cc6081e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:26:52,963 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:26:52,963 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75255721{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:26:52,963 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@219c70cc{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,STOPPED} 2024-11-26T23:26:52,964 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:26:52,965 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:26:52,965 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1916363482-172.17.0.2-1732663594093 (Datanode Uuid 75864b72-31eb-4f65-97d9-6a407e9e7835) service to localhost/127.0.0.1:34607 2024-11-26T23:26:52,965 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:26:52,965 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data5/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:52,966 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data6/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:26:52,966 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:26:52,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40529 {}] regionserver.HRegion(8855): Flush requested on 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:52,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6644eb6ad870b0b346267b71ec1a8f5b 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-26T23:26:52,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/d47c68717e064c25a69eb8593329bf5f is 1080, key is row0002/info:/1732663608954/Put/seqid=0 2024-11-26T23:26:52,996 WARN [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741840_1023 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39209 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:52,996 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47188 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741840_1023] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741840_1023 to mirror 127.0.0.1:39209 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:52,996 WARN [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741840_1023 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:26:52,996 WARN [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741840_1023 2024-11-26T23:26:52,996 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47188 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741840_1023] {}] datanode.BlockReceiver(316): Block 1073741840 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:52,996 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47188 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741840_1023] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47188 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:52,999 WARN [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:26:53,001 WARN [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741841_1024 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:53,002 WARN [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741841_1024 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK], DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]) is bad. 2024-11-26T23:26:53,002 WARN [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741841_1024 2024-11-26T23:26:53,002 WARN [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK] 2024-11-26T23:26:53,003 WARN [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741842_1025 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:53,003 WARN [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741842_1025 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK], DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]) is bad. 2024-11-26T23:26:53,003 WARN [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741842_1025 2024-11-26T23:26:53,003 WARN [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK] 2024-11-26T23:26:53,005 WARN [Thread-919 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741843_1026 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:53,005 WARN [Thread-919 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741843_1026 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:53,005 WARN [Thread-919 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741843_1026 2024-11-26T23:26:53,005 WARN [Thread-919 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK] 2024-11-26T23:26:53,006 WARN [IPC Server handler 0 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-26T23:26:53,006 WARN [IPC Server handler 0 on default port 34607 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-26T23:26:53,006 WARN [IPC Server handler 0 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-26T23:26:53,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741844_1027 (size=10347) 2024-11-26T23:26:53,220 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:53,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/d47c68717e064c25a69eb8593329bf5f 2024-11-26T23:26:53,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/d47c68717e064c25a69eb8593329bf5f as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/d47c68717e064c25a69eb8593329bf5f 2024-11-26T23:26:53,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/d47c68717e064c25a69eb8593329bf5f, entries=5, sequenceid=11, filesize=10.1 K 2024-11-26T23:26:53,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=9.45 KB/9681 for 6644eb6ad870b0b346267b71ec1a8f5b in 450ms, sequenceid=11, compaction requested=false 2024-11-26T23:26:53,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6644eb6ad870b0b346267b71ec1a8f5b: 2024-11-26T23:26:53,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40529 {}] regionserver.HRegion(8855): Flush requested on 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:53,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6644eb6ad870b0b346267b71ec1a8f5b 1/1 column families, dataSize=10.50 KB heapSize=11.50 KB 2024-11-26T23:26:53,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/9cfb82063a5c4b3d90fc301693ce1d4f is 1080, key is row0007/info:/1732663612975/Put/seqid=0 2024-11-26T23:26:53,599 WARN [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741845_1028 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:53,599 WARN [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741845_1028 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK], DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]) is bad. 2024-11-26T23:26:53,599 WARN [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741845_1028 2024-11-26T23:26:53,600 WARN [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK] 2024-11-26T23:26:53,601 WARN [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741846_1029 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:53,601 WARN [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741846_1029 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK], DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:53,601 WARN [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741846_1029 2024-11-26T23:26:53,602 WARN [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK] 2024-11-26T23:26:53,604 WARN [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741847_1030 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39209 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:53,604 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47198 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741847_1030] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741847_1030 to mirror 127.0.0.1:39209 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:53,604 WARN [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741847_1030 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:26:53,604 WARN [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741847_1030 2024-11-26T23:26:53,604 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47198 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741847_1030] {}] datanode.BlockReceiver(316): Block 1073741847 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:53,605 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47198 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741847_1030] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47198 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:53,605 WARN [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:26:53,607 WARN [Thread-925 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741848_1031 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34663 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:53,607 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47208 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741848_1031] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741848_1031 to mirror 127.0.0.1:34663 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:53,607 WARN [Thread-925 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741848_1031 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]) is bad. 2024-11-26T23:26:53,607 WARN [Thread-925 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741848_1031 2024-11-26T23:26:53,607 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47208 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741848_1031] {}] datanode.BlockReceiver(316): Block 1073741848 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:53,607 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47208 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741848_1031] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47208 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:53,607 WARN [Thread-925 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK] 2024-11-26T23:26:53,608 WARN [IPC Server handler 0 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-26T23:26:53,608 WARN [IPC Server handler 0 on default port 34607 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-26T23:26:53,608 WARN [IPC Server handler 0 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-26T23:26:53,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741849_1032 (size=12506) 2024-11-26T23:26:54,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.50 KB at sequenceid=24 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/9cfb82063a5c4b3d90fc301693ce1d4f 2024-11-26T23:26:54,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/9cfb82063a5c4b3d90fc301693ce1d4f as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/9cfb82063a5c4b3d90fc301693ce1d4f 2024-11-26T23:26:54,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/9cfb82063a5c4b3d90fc301693ce1d4f, entries=7, sequenceid=24, filesize=12.2 K 2024-11-26T23:26:54,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.50 KB/10756, heapSize ~11.48 KB/11760, currentSize=2.10 KB/2150 for 6644eb6ad870b0b346267b71ec1a8f5b in 432ms, sequenceid=24, compaction requested=false 2024-11-26T23:26:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6644eb6ad870b0b346267b71ec1a8f5b: 2024-11-26T23:26:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=22.3 K, sizeToCheck=16.0 K 2024-11-26T23:26:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:54,026 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/9cfb82063a5c4b3d90fc301693ce1d4f because midkey is the same as first or last row 2024-11-26T23:26:54,150 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:54,952 WARN [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]] 2024-11-26T23:26:54,953 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:54,953 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C40529%2C1732663595141:(num 1732663610944) roll requested 2024-11-26T23:26:54,953 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C40529%2C1732663595141.1732663614953 2024-11-26T23:26:54,956 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741850_1033 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:54,956 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741850_1033 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK], DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:54,956 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741850_1033 2024-11-26T23:26:54,957 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK] 2024-11-26T23:26:54,959 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741851_1034 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34663 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:54,959 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47224 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741851_1034] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741851_1034 to mirror 127.0.0.1:34663 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:54,959 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741851_1034 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]) is bad. 2024-11-26T23:26:54,959 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741851_1034 2024-11-26T23:26:54,959 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47224 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741851_1034] {}] datanode.BlockReceiver(316): Block 1073741851 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-26T23:26:54,959 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47224 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741851_1034] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47224 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:54,960 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK] 2024-11-26T23:26:54,961 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741852_1035 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:54,961 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741852_1035 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK], DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]) is bad. 2024-11-26T23:26:54,961 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741852_1035 2024-11-26T23:26:54,961 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK] 2024-11-26T23:26:54,962 WARN [Thread-931 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741853_1036 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:54,963 WARN [Thread-931 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741853_1036 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:26:54,963 WARN [Thread-931 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741853_1036 2024-11-26T23:26:54,963 WARN [Thread-931 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:26:54,964 WARN [IPC Server handler 4 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-26T23:26:54,964 WARN [IPC Server handler 4 on default port 34607 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-26T23:26:54,964 WARN [IPC Server handler 4 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-26T23:26:54,966 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:54,967 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:54,967 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:54,967 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:54,967 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:54,967 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663610944 with entries=25, filesize=25.38 KB; new WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663614953 2024-11-26T23:26:54,969 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34309:34309)] 2024-11-26T23:26:54,969 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 is not closed yet, will try archiving it next time 2024-11-26T23:26:54,969 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663610944 is not closed yet, will try archiving it next time 2024-11-26T23:26:54,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741839_1022 (size=25992) 2024-11-26T23:26:54,970 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663606935 to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs/c707e9ace675%2C40529%2C1732663595141.1732663606935 2024-11-26T23:26:55,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40529 {}] regionserver.HRegion(8855): Flush requested on 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:55,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6644eb6ad870b0b346267b71ec1a8f5b 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-26T23:26:55,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/7a41735b06364bc7a435a31c1bee62c0 is 1079, key is tmprow/info:/1732663615011/Put/seqid=0 2024-11-26T23:26:55,018 WARN [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741855_1038 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:55,018 WARN [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741855_1038 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]) is bad. 2024-11-26T23:26:55,018 WARN [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741855_1038 2024-11-26T23:26:55,019 WARN [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK] 2024-11-26T23:26:55,020 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47228 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741856_1039] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741856_1039 to mirror 127.0.0.1:34663 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,021 WARN [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741856_1039 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34663 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:55,021 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47228 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741856_1039] {}] datanode.BlockReceiver(316): Block 1073741856 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:55,021 WARN [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741856_1039 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]) is bad. 2024-11-26T23:26:55,021 WARN [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741856_1039 2024-11-26T23:26:55,021 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47228 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741856_1039] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47228 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,021 WARN [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK] 2024-11-26T23:26:55,022 WARN [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741857_1040 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:55,023 WARN [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741857_1040 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:26:55,023 WARN [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741857_1040 2024-11-26T23:26:55,023 WARN [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:26:55,024 WARN [Thread-936 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741858_1041 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:55,024 WARN [Thread-936 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741858_1041 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:55,024 WARN [Thread-936 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741858_1041 2024-11-26T23:26:55,025 WARN [Thread-936 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK] 2024-11-26T23:26:55,025 WARN [IPC Server handler 0 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-26T23:26:55,025 WARN [IPC Server handler 0 on default port 34607 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-26T23:26:55,025 WARN [IPC Server handler 0 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-26T23:26:55,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741859_1042 (size=6027) 2024-11-26T23:26:55,220 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:55,370 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 is not closed yet, will try archiving it next time 2024-11-26T23:26:55,429 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=34 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/7a41735b06364bc7a435a31c1bee62c0 2024-11-26T23:26:55,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/7a41735b06364bc7a435a31c1bee62c0 as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/7a41735b06364bc7a435a31c1bee62c0 2024-11-26T23:26:55,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/7a41735b06364bc7a435a31c1bee62c0, entries=1, sequenceid=34, filesize=5.9 K 2024-11-26T23:26:55,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 6644eb6ad870b0b346267b71ec1a8f5b in 430ms, sequenceid=34, compaction requested=true 2024-11-26T23:26:55,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6644eb6ad870b0b346267b71ec1a8f5b: 2024-11-26T23:26:55,442 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.2 K, sizeToCheck=16.0 K 2024-11-26T23:26:55,442 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:55,442 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/9cfb82063a5c4b3d90fc301693ce1d4f because midkey is the same as first or last row 2024-11-26T23:26:55,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6644eb6ad870b0b346267b71ec1a8f5b:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:26:55,443 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:26:55,443 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:26:55,444 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 28880 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:26:55,444 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HStore(1541): 6644eb6ad870b0b346267b71ec1a8f5b/info is initiating minor compaction (all files) 2024-11-26T23:26:55,444 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6644eb6ad870b0b346267b71ec1a8f5b/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:26:55,444 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/d47c68717e064c25a69eb8593329bf5f, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/9cfb82063a5c4b3d90fc301693ce1d4f, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/7a41735b06364bc7a435a31c1bee62c0] into tmpdir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp, totalSize=28.2 K 2024-11-26T23:26:55,445 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.Compactor(225): Compacting d47c68717e064c25a69eb8593329bf5f, keycount=5, bloomtype=ROW, size=10.1 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732663608954 2024-11-26T23:26:55,445 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cfb82063a5c4b3d90fc301693ce1d4f, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=24, earliestPutTs=1732663612975 2024-11-26T23:26:55,446 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a41735b06364bc7a435a31c1bee62c0, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732663615011 2024-11-26T23:26:55,464 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6644eb6ad870b0b346267b71ec1a8f5b#info#compaction#21 average throughput is 6.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:26:55,464 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/64272a46662d4b5a9434f8c621c8db28 is 1080, key is row0002/info:/1732663608954/Put/seqid=0 2024-11-26T23:26:55,467 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741860_1043 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44117 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:55,467 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47268 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741860_1043] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741860_1043 to mirror 127.0.0.1:44117 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,467 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741860_1043 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]) is bad. 2024-11-26T23:26:55,467 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741860_1043 2024-11-26T23:26:55,468 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47268 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741860_1043] {}] datanode.BlockReceiver(316): Block 1073741860 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:55,468 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47268 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741860_1043] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47268 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,468 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK] 2024-11-26T23:26:55,471 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741861_1044 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34663 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:55,471 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47276 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741861_1044] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741861_1044 to mirror 127.0.0.1:34663 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,471 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741861_1044 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]) is bad. 2024-11-26T23:26:55,471 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741861_1044 2024-11-26T23:26:55,471 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47276 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741861_1044] {}] datanode.BlockReceiver(316): Block 1073741861 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:55,471 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47276 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741861_1044] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47276 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,471 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK] 2024-11-26T23:26:55,473 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741862_1045 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39209 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:55,473 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47290 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741862_1045] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741862_1045 to mirror 127.0.0.1:39209 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,474 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741862_1045 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:26:55,474 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741862_1045 2024-11-26T23:26:55,474 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47290 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741862_1045] {}] datanode.BlockReceiver(316): Block 1073741862 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:55,474 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47290 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741862_1045] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47290 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,474 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:26:55,476 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47300 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741863_1046] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741863_1046 to mirror 127.0.0.1:40289 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,476 WARN [Thread-943 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741863_1046 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40289 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:55,477 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47300 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741863_1046] {}] datanode.BlockReceiver(316): Block 1073741863 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:55,477 WARN [Thread-943 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741863_1046 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:55,477 WARN [Thread-943 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741863_1046 2024-11-26T23:26:55,477 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47300 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741863_1046] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47300 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,477 WARN [Thread-943 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK] 2024-11-26T23:26:55,478 WARN [IPC Server handler 2 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-26T23:26:55,478 WARN [IPC Server handler 2 on default port 34607 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-26T23:26:55,478 WARN [IPC Server handler 2 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-26T23:26:55,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741864_1047 (size=17994) 2024-11-26T23:26:55,660 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6ccf02df[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741844_1027 to 127.0.0.1:40289 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,660 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6208e136[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741849_1032 to 127.0.0.1:40289 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:55,889 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/64272a46662d4b5a9434f8c621c8db28 as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/64272a46662d4b5a9434f8c621c8db28 2024-11-26T23:26:55,897 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6644eb6ad870b0b346267b71ec1a8f5b/info of 6644eb6ad870b0b346267b71ec1a8f5b into 64272a46662d4b5a9434f8c621c8db28(size=17.6 K), total size for store is 17.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:26:55,897 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6644eb6ad870b0b346267b71ec1a8f5b: 2024-11-26T23:26:55,897 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b., storeName=6644eb6ad870b0b346267b71ec1a8f5b/info, priority=13, startTime=1732663615442; duration=0sec 2024-11-26T23:26:55,898 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-26T23:26:55,898 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:55,898 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/64272a46662d4b5a9434f8c621c8db28 because midkey is the same as first or last row 2024-11-26T23:26:55,898 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-26T23:26:55,898 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:55,898 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/64272a46662d4b5a9434f8c621c8db28 because midkey is the same as first or last row 2024-11-26T23:26:55,898 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.6 K, sizeToCheck=16.0 K 2024-11-26T23:26:55,898 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:55,898 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/64272a46662d4b5a9434f8c621c8db28 because midkey is the same as first or last row 2024-11-26T23:26:55,898 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:26:55,898 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6644eb6ad870b0b346267b71ec1a8f5b:info 2024-11-26T23:26:56,150 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:56,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40529 {}] regionserver.HRegion(8855): Flush requested on 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:56,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6644eb6ad870b0b346267b71ec1a8f5b 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-26T23:26:56,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/ccd2a739c89c4078bf29d3e694ff4f12 is 1079, key is tmprow/info:/1732663616430/Put/seqid=0 2024-11-26T23:26:56,437 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741865_1048 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:56,438 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741865_1048 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:56,438 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741865_1048 2024-11-26T23:26:56,438 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK] 2024-11-26T23:26:56,439 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741866_1049 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:56,439 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741866_1049 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]) is bad. 2024-11-26T23:26:56,439 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741866_1049 2024-11-26T23:26:56,440 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK] 2024-11-26T23:26:56,442 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741867_1050 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:34663 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:56,442 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47312 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741867_1050] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741867_1050 to mirror 127.0.0.1:34663 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:56,442 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741867_1050 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]) is bad. 2024-11-26T23:26:56,442 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741867_1050 2024-11-26T23:26:56,442 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47312 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741867_1050] {}] datanode.BlockReceiver(316): Block 1073741867 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:56,442 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47312 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741867_1050] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47312 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:56,442 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK] 2024-11-26T23:26:56,443 WARN [Thread-955 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741868_1051 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:56,443 WARN [Thread-955 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741868_1051 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:26:56,443 WARN [Thread-955 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741868_1051 2024-11-26T23:26:56,444 WARN [Thread-955 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:26:56,444 WARN [IPC Server handler 4 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-26T23:26:56,444 WARN [IPC Server handler 4 on default port 34607 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-26T23:26:56,444 WARN [IPC Server handler 4 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-26T23:26:56,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741869_1052 (size=6027) 2024-11-26T23:26:56,660 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6ccf02df[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741859_1042 to 127.0.0.1:34663 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:56,660 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6208e136[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741839_1022 to 127.0.0.1:44117 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:56,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/ccd2a739c89c4078bf29d3e694ff4f12 2024-11-26T23:26:56,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/ccd2a739c89c4078bf29d3e694ff4f12 as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/ccd2a739c89c4078bf29d3e694ff4f12 2024-11-26T23:26:56,865 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/ccd2a739c89c4078bf29d3e694ff4f12, entries=1, sequenceid=45, filesize=5.9 K 2024-11-26T23:26:56,866 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 6644eb6ad870b0b346267b71ec1a8f5b in 435ms, sequenceid=45, compaction requested=false 2024-11-26T23:26:56,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6644eb6ad870b0b346267b71ec1a8f5b: 2024-11-26T23:26:56,866 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=23.5 K, sizeToCheck=16.0 K 2024-11-26T23:26:56,866 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:56,867 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/64272a46662d4b5a9434f8c621c8db28 because midkey is the same as first or last row 2024-11-26T23:26:56,970 WARN [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(529): HDFS pipeline error detected. Found 1 replicas but expecting no less than 2 replicas. Requesting close of WAL. current pipeline: [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]] 2024-11-26T23:26:56,970 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:56,970 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C40529%2C1732663595141:(num 1732663614953) roll requested 2024-11-26T23:26:56,970 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C40529%2C1732663595141.1732663616970 2024-11-26T23:26:56,974 WARN [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741870_1053 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44117 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:56,974 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47342 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741870_1053] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741870_1053 to mirror 127.0.0.1:44117 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:56,974 WARN [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741870_1053 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]) is bad. 2024-11-26T23:26:56,974 WARN [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741870_1053 2024-11-26T23:26:56,974 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47342 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741870_1053] {}] datanode.BlockReceiver(316): Block 1073741870 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-26T23:26:56,974 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47342 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741870_1053] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47342 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:56,975 WARN [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK] 2024-11-26T23:26:56,977 WARN [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741871_1054 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39209 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:56,976 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47354 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741871_1054] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741871_1054 to mirror 127.0.0.1:39209 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:56,977 WARN [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741871_1054 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:26:56,977 WARN [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741871_1054 2024-11-26T23:26:56,977 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47354 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741871_1054] {}] datanode.BlockReceiver(316): Block 1073741871 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-26T23:26:56,977 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47354 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741871_1054] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47354 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:56,977 WARN [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:26:56,978 WARN [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741872_1055 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:56,978 WARN [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741872_1055 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]) is bad. 2024-11-26T23:26:56,978 WARN [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741872_1055 2024-11-26T23:26:56,979 WARN [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK] 2024-11-26T23:26:56,980 WARN [Thread-960 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741873_1056 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:56,980 WARN [Thread-960 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741873_1056 in pipeline [DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:56,980 WARN [Thread-960 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741873_1056 2024-11-26T23:26:56,980 WARN [Thread-960 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK] 2024-11-26T23:26:56,981 WARN [IPC Server handler 4 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-26T23:26:56,981 WARN [IPC Server handler 4 on default port 34607 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-26T23:26:56,981 WARN [IPC Server handler 4 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-26T23:26:56,983 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:56,983 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:56,983 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:56,984 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:56,984 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:26:56,984 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663614953 with entries=15, filesize=13.26 KB; new WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663616970 2024-11-26T23:26:56,985 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34309:34309)] 2024-11-26T23:26:56,985 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 is not closed yet, will try archiving it next time 2024-11-26T23:26:56,985 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663614953 is not closed yet, will try archiving it next time 2024-11-26T23:26:56,985 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663610944 to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs/c707e9ace675%2C40529%2C1732663595141.1732663610944 2024-11-26T23:26:56,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741854_1037 (size=13591) 2024-11-26T23:26:57,221 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:57,386 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 is not closed yet, will try archiving it next time 2024-11-26T23:26:57,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40529 {}] regionserver.HRegion(8855): Flush requested on 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:26:57,850 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6644eb6ad870b0b346267b71ec1a8f5b 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-26T23:26:57,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/b35d319b80f140578c9e6752aab450fe is 1079, key is tmprow/info:/1732663617849/Put/seqid=0 2024-11-26T23:26:57,856 WARN [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741875_1058 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:57,857 WARN [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741875_1058 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK], DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:26:57,857 WARN [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741875_1058 2024-11-26T23:26:57,857 WARN [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:26:57,858 WARN [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741876_1059 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:57,858 WARN [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741876_1059 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK], DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]) is bad. 2024-11-26T23:26:57,858 WARN [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741876_1059 2024-11-26T23:26:57,859 WARN [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK] 2024-11-26T23:26:57,861 WARN [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741877_1060 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40289 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:57,861 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47358 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741877_1060] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741877_1060 to mirror 127.0.0.1:40289 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:57,861 WARN [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741877_1060 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:57,861 WARN [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741877_1060 2024-11-26T23:26:57,861 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47358 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741877_1060] {}] datanode.BlockReceiver(316): Block 1073741877 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:57,861 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47358 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741877_1060] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47358 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:57,862 WARN [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK] 2024-11-26T23:26:57,863 WARN [Thread-966 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741878_1061 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:57,863 WARN [Thread-966 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741878_1061 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]) is bad. 2024-11-26T23:26:57,863 WARN [Thread-966 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741878_1061 2024-11-26T23:26:57,864 WARN [Thread-966 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK] 2024-11-26T23:26:57,864 WARN [IPC Server handler 1 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-26T23:26:57,864 WARN [IPC Server handler 1 on default port 34607 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-26T23:26:57,865 WARN [IPC Server handler 1 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-26T23:26:57,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741879_1062 (size=6027) 2024-11-26T23:26:58,151 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:58,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/b35d319b80f140578c9e6752aab450fe 2024-11-26T23:26:58,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/b35d319b80f140578c9e6752aab450fe as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/b35d319b80f140578c9e6752aab450fe 2024-11-26T23:26:58,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/b35d319b80f140578c9e6752aab450fe, entries=1, sequenceid=55, filesize=5.9 K 2024-11-26T23:26:58,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7525, heapSize ~8.11 KB/8304, currentSize=2.10 KB/2150 for 6644eb6ad870b0b346267b71ec1a8f5b in 431ms, sequenceid=55, compaction requested=true 2024-11-26T23:26:58,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6644eb6ad870b0b346267b71ec1a8f5b: 2024-11-26T23:26:58,281 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=29.3 K, sizeToCheck=16.0 K 2024-11-26T23:26:58,281 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:58,282 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/64272a46662d4b5a9434f8c621c8db28 because midkey is the same as first or last row 2024-11-26T23:26:58,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6644eb6ad870b0b346267b71ec1a8f5b:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:26:58,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:26:58,282 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:26:58,283 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 30048 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:26:58,283 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HStore(1541): 6644eb6ad870b0b346267b71ec1a8f5b/info is initiating minor compaction (all files) 2024-11-26T23:26:58,283 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6644eb6ad870b0b346267b71ec1a8f5b/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:26:58,283 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/64272a46662d4b5a9434f8c621c8db28, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/ccd2a739c89c4078bf29d3e694ff4f12, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/b35d319b80f140578c9e6752aab450fe] into tmpdir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp, totalSize=29.3 K 2024-11-26T23:26:58,283 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64272a46662d4b5a9434f8c621c8db28, keycount=12, bloomtype=ROW, size=17.6 K, encoding=NONE, compression=NONE, seqNum=34, earliestPutTs=1732663608954 2024-11-26T23:26:58,284 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccd2a739c89c4078bf29d3e694ff4f12, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732663616430 2024-11-26T23:26:58,284 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.Compactor(225): Compacting b35d319b80f140578c9e6752aab450fe, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732663617849 2024-11-26T23:26:58,300 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6644eb6ad870b0b346267b71ec1a8f5b#info#compaction#24 average throughput is 4.10 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:26:58,300 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/87fc198dd8394089b21381d4038378f5 is 1080, key is row0002/info:/1732663608954/Put/seqid=0 2024-11-26T23:26:58,303 WARN [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741880_1063 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:44117 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:58,303 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47390 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741880_1063] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741880_1063 to mirror 127.0.0.1:44117 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:58,303 WARN [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741880_1063 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK]) is bad. 2024-11-26T23:26:58,303 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47390 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741880_1063] {}] datanode.BlockReceiver(316): Block 1073741880 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:58,303 WARN [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741880_1063 2024-11-26T23:26:58,303 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47390 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741880_1063] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47390 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:58,304 WARN [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:44117,DS-550c15a6-fa37-4e21-8f11-28a425daf14b,DISK] 2024-11-26T23:26:58,305 WARN [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741881_1064 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:58,305 WARN [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741881_1064 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK], DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:26:58,305 WARN [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741881_1064 2024-11-26T23:26:58,306 WARN [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:26:58,308 WARN [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741882_1065 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:40289 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:58,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47392 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741882_1065] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741882_1065 to mirror 127.0.0.1:40289 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:58,308 WARN [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741882_1065 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]) is bad. 2024-11-26T23:26:58,308 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47392 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741882_1065] {}] datanode.BlockReceiver(316): Block 1073741882 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:26:58,308 WARN [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741882_1065 2024-11-26T23:26:58,308 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:47392 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741882_1065] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47392 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:58,309 WARN [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:40289,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK] 2024-11-26T23:26:58,310 WARN [Thread-971 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741883_1066 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:58,310 WARN [Thread-971 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741883_1066 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]) is bad. 2024-11-26T23:26:58,310 WARN [Thread-971 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741883_1066 2024-11-26T23:26:58,311 WARN [Thread-971 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK] 2024-11-26T23:26:58,311 WARN [IPC Server handler 4 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-26T23:26:58,312 WARN [IPC Server handler 4 on default port 34607 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=2, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-26T23:26:58,312 WARN [IPC Server handler 4 on default port 34607 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 2 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-26T23:26:58,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741884_1067 (size=18097) 2024-11-26T23:26:58,660 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6ccf02df[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741864_1047 to 127.0.0.1:34663 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:58,660 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6208e136[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741869_1052 to 127.0.0.1:39209 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:58,722 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/87fc198dd8394089b21381d4038378f5 as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/87fc198dd8394089b21381d4038378f5 2024-11-26T23:26:58,728 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6644eb6ad870b0b346267b71ec1a8f5b/info of 6644eb6ad870b0b346267b71ec1a8f5b into 87fc198dd8394089b21381d4038378f5(size=17.7 K), total size for store is 17.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:26:58,728 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6644eb6ad870b0b346267b71ec1a8f5b: 2024-11-26T23:26:58,728 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b., storeName=6644eb6ad870b0b346267b71ec1a8f5b/info, priority=13, startTime=1732663618282; duration=0sec 2024-11-26T23:26:58,729 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-26T23:26:58,729 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:58,729 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/87fc198dd8394089b21381d4038378f5 because midkey is the same as first or last row 2024-11-26T23:26:58,729 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-26T23:26:58,729 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:58,729 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/87fc198dd8394089b21381d4038378f5 because midkey is the same as first or last row 2024-11-26T23:26:58,729 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=17.7 K, sizeToCheck=16.0 K 2024-11-26T23:26:58,729 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:26:58,729 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/87fc198dd8394089b21381d4038378f5 because midkey is the same as first or last row 2024-11-26T23:26:58,729 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:26:58,729 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6644eb6ad870b0b346267b71ec1a8f5b:info 2024-11-26T23:26:58,985 WARN [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(539): Too many consecutive RollWriter requests, it's a sign of the total number of live datanodes is lower than the tolerable replicas. 2024-11-26T23:26:58,985 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:59,073 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:26:59,076 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:26:59,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:26:59,077 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:26:59,077 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:26:59,077 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@13c2f5a4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:26:59,078 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1191c470{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:26:59,192 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d95479b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/java.io.tmpdir/jetty-localhost-34667-hadoop-hdfs-3_4_1-tests_jar-_-any-8480032948569484972/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:26:59,192 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@469c1f5d{HTTP/1.1, (http/1.1)}{localhost:34667} 2024-11-26T23:26:59,193 INFO [Time-limited test {}] server.Server(415): Started @128537ms 2024-11-26T23:26:59,194 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:26:59,221 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:26:59,289 WARN [Thread-991 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:26:59,297 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfb15d79f8cf73154 with lease ID 0x9f3d198b30680ff9: from storage DS-cacf0762-8e58-4aa7-98b9-88feb8cde337 node DatanodeRegistration(127.0.0.1:36419, datanodeUuid=957de82d-0af5-4420-8b0f-1112e576e3dd, infoPort=33751, infoSecurePort=0, ipcPort=37035, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:59,298 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfb15d79f8cf73154 with lease ID 0x9f3d198b30680ff9: from storage DS-0b538a44-15c7-42ec-a0e7-fd2bd11044d9 node DatanodeRegistration(127.0.0.1:36419, datanodeUuid=957de82d-0af5-4420-8b0f-1112e576e3dd, infoPort=33751, infoSecurePort=0, ipcPort=37035, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:26:59,661 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6ccf02df[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741879_1062 to 127.0.0.1:39209 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:26:59,661 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@6208e136[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:44973, datanodeUuid=c2153758-3f66-4b49-9a42-9c8c6e787d90, infoPort=34309, infoSecurePort=0, ipcPort=35337, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741854_1037 to 127.0.0.1:39209 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:00,151 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:00,985 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:01,221 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:01,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741884_1067 (size=18097) 2024-11-26T23:27:02,151 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:02,986 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:03,222 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:04,152 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:04,974 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:27:04,986 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:05,222 INFO [master:store-WAL-Roller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:05,323 ERROR [FSHLog-0-hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData-prefix:c707e9ace675,33077,1732663595034 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:05,323 WARN [FSHLog-0-hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData-prefix:c707e9ace675,33077,1732663595034 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:05,323 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C33077%2C1732663595034:(num 1732663595240) roll requested 2024-11-26T23:27:05,323 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C33077%2C1732663595034.1732663625323 2024-11-26T23:27:05,329 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:05,330 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:05,330 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:05,330 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:05,330 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:05,330 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034/c707e9ace675%2C33077%2C1732663595034.1732663595240 with entries=54, filesize=26.67 KB; new WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034/c707e9ace675%2C33077%2C1732663595034.1732663625323 2024-11-26T23:27:05,330 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:05,331 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:05,331 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034/c707e9ace675%2C33077%2C1732663595034.1732663595240 2024-11-26T23:27:05,331 WARN [IPC Server handler 1 on default port 34607 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034/c707e9ace675%2C33077%2C1732663595034.1732663595240 has not been closed. Lease recovery is in progress. RecoveryId = 1069 for block blk_1073741830_1006 2024-11-26T23:27:05,331 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034/c707e9ace675%2C33077%2C1732663595034.1732663595240 after 0ms 2024-11-26T23:27:05,332 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33751:33751),(127.0.0.1/127.0.0.1:34309:34309)] 2024-11-26T23:27:05,332 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034/c707e9ace675%2C33077%2C1732663595034.1732663595240 is not closed yet, will try archiving it next time 2024-11-26T23:27:06,152 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:06,987 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:08,152 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:08,987 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:09,316 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@271ff464 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1916363482-172.17.0.2-1732663594093:blk_1073741833_1009, datanode=DatanodeInfoWithStorage[127.0.0.1:34663,null,null]) java.net.ConnectException: Call From c707e9ace675/172.17.0.2 to localhost:44355 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-26T23:27:09,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741833_1019 (size=455) 2024-11-26T23:27:09,332 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034/c707e9ace675%2C33077%2C1732663595034.1732663595240 after 4001ms 2024-11-26T23:27:09,967 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663595602 to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs/c707e9ace675%2C40529%2C1732663595141.1732663595602 2024-11-26T23:27:09,969 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663614953 to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs/c707e9ace675%2C40529%2C1732663595141.1732663614953 2024-11-26T23:27:10,153 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:10,987 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:11,292 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@1672b392[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=957de82d-0af5-4420-8b0f-1112e576e3dd, infoPort=33751, infoSecurePort=0, ipcPort=37035, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741833_1019 to 127.0.0.1:39209 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:11,292 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@1f546071[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=957de82d-0af5-4420-8b0f-1112e576e3dd, infoPort=33751, infoSecurePort=0, ipcPort=37035, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741835_1011 to 127.0.0.1:39209 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:12,153 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:12,292 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@1672b392[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=957de82d-0af5-4420-8b0f-1112e576e3dd, infoPort=33751, infoSecurePort=0, ipcPort=37035, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741829_1005 to 127.0.0.1:39209 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:12,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:27:12,510 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C40529%2C1732663595141.1732663632510 2024-11-26T23:27:12,516 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:12,516 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:12,516 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:12,516 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:12,516 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:12,517 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663616970 with entries=13, filesize=12.60 KB; new WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663632510 2024-11-26T23:27:12,517 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33751:33751),(127.0.0.1/127.0.0.1:34309:34309)] 2024-11-26T23:27:12,518 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663616970 is not closed yet, will try archiving it next time 2024-11-26T23:27:12,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741874_1057 (size=12911) 2024-11-26T23:27:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40529 {}] regionserver.HRegion(8855): Flush requested on 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:27:12,522 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6644eb6ad870b0b346267b71ec1a8f5b 1/1 column families, dataSize=7.35 KB heapSize=8.13 KB 2024-11-26T23:27:12,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/6d0ff4c365de4853b2016846606f547f is 1080, key is row0013/info:/1732663632519/Put/seqid=0 2024-11-26T23:27:12,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741887_1071 (size=8190) 2024-11-26T23:27:12,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741887_1071 (size=8190) 2024-11-26T23:27:12,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.35 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/6d0ff4c365de4853b2016846606f547f 2024-11-26T23:27:12,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/6d0ff4c365de4853b2016846606f547f as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/6d0ff4c365de4853b2016846606f547f 2024-11-26T23:27:12,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/6d0ff4c365de4853b2016846606f547f, entries=3, sequenceid=66, filesize=8.0 K 2024-11-26T23:27:12,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.35 KB/7527, heapSize ~8.11 KB/8304, currentSize=9.46 KB/9683 for 6644eb6ad870b0b346267b71ec1a8f5b in 35ms, sequenceid=66, compaction requested=false 2024-11-26T23:27:12,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6644eb6ad870b0b346267b71ec1a8f5b: 2024-11-26T23:27:12,557 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=25.7 K, sizeToCheck=16.0 K 2024-11-26T23:27:12,557 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:27:12,557 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/87fc198dd8394089b21381d4038378f5 because midkey is the same as first or last row 2024-11-26T23:27:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40529 {}] regionserver.HRegion(8855): Flush requested on 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:27:12,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6644eb6ad870b0b346267b71ec1a8f5b 1/1 column families, dataSize=10.51 KB heapSize=11.50 KB 2024-11-26T23:27:12,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/22aa6f4cd4ac41a4883573932f41190a is 1080, key is row0015/info:/1732663632523/Put/seqid=0 2024-11-26T23:27:12,751 WARN [Thread-1038 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741888_1072 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39209 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:12,751 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:57038 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741888_1072] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741888_1072 to mirror 127.0.0.1:39209 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:12,751 WARN [Thread-1038 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741888_1072 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:27:12,751 WARN [Thread-1038 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741888_1072 2024-11-26T23:27:12,751 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:57038 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741888_1072] {}] datanode.BlockReceiver(316): Block 1073741888 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:27:12,751 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:57038 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741888_1072] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57038 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:12,752 WARN [Thread-1038 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:27:12,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741889_1073 (size=14660) 2024-11-26T23:27:12,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741889_1073 (size=14660) 2024-11-26T23:27:12,757 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.51 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/22aa6f4cd4ac41a4883573932f41190a 2024-11-26T23:27:12,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/22aa6f4cd4ac41a4883573932f41190a as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/22aa6f4cd4ac41a4883573932f41190a 2024-11-26T23:27:12,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/22aa6f4cd4ac41a4883573932f41190a, entries=9, sequenceid=79, filesize=14.3 K 2024-11-26T23:27:12,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~10.51 KB/10758, heapSize ~11.48 KB/11760, currentSize=0 B/0 for 6644eb6ad870b0b346267b71ec1a8f5b in 26ms, sequenceid=79, compaction requested=true 2024-11-26T23:27:12,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6644eb6ad870b0b346267b71ec1a8f5b: 2024-11-26T23:27:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=40.0 K, sizeToCheck=16.0 K 2024-11-26T23:27:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:27:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/87fc198dd8394089b21381d4038378f5 because midkey is the same as first or last row 2024-11-26T23:27:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6644eb6ad870b0b346267b71ec1a8f5b:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:27:12,771 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:27:12,771 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:27:12,772 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40947 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:27:12,772 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HStore(1541): 6644eb6ad870b0b346267b71ec1a8f5b/info is initiating minor compaction (all files) 2024-11-26T23:27:12,772 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6644eb6ad870b0b346267b71ec1a8f5b/info in TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:27:12,772 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/87fc198dd8394089b21381d4038378f5, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/6d0ff4c365de4853b2016846606f547f, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/22aa6f4cd4ac41a4883573932f41190a] into tmpdir=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp, totalSize=40.0 K 2024-11-26T23:27:12,773 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87fc198dd8394089b21381d4038378f5, keycount=12, bloomtype=ROW, size=17.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732663608954 2024-11-26T23:27:12,773 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d0ff4c365de4853b2016846606f547f, keycount=3, bloomtype=ROW, size=8.0 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1732663618863 2024-11-26T23:27:12,774 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22aa6f4cd4ac41a4883573932f41190a, keycount=9, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732663632523 2024-11-26T23:27:12,786 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6644eb6ad870b0b346267b71ec1a8f5b#info#compaction#27 average throughput is 22.57 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:27:12,787 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/d089b6acdd2641cd9ef6a0065dda35bc is 1080, key is row0002/info:/1732663608954/Put/seqid=0 2024-11-26T23:27:12,788 WARN [Thread-1046 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741890_1074 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:12,789 WARN [Thread-1046 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741890_1074 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:27:12,789 WARN [Thread-1046 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741890_1074 2024-11-26T23:27:12,789 WARN [Thread-1046 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:27:12,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741891_1075 (size=28989) 2024-11-26T23:27:12,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741891_1075 (size=28989) 2024-11-26T23:27:12,801 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/d089b6acdd2641cd9ef6a0065dda35bc as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/d089b6acdd2641cd9ef6a0065dda35bc 2024-11-26T23:27:12,808 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6644eb6ad870b0b346267b71ec1a8f5b/info of 6644eb6ad870b0b346267b71ec1a8f5b into d089b6acdd2641cd9ef6a0065dda35bc(size=28.3 K), total size for store is 28.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:27:12,808 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6644eb6ad870b0b346267b71ec1a8f5b: 2024-11-26T23:27:12,808 INFO [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b., storeName=6644eb6ad870b0b346267b71ec1a8f5b/info, priority=13, startTime=1732663632771; duration=0sec 2024-11-26T23:27:12,809 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-11-26T23:27:12,809 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:27:12,809 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/d089b6acdd2641cd9ef6a0065dda35bc because midkey is the same as first or last row 2024-11-26T23:27:12,809 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-11-26T23:27:12,809 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:27:12,809 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/d089b6acdd2641cd9ef6a0065dda35bc because midkey is the same as first or last row 2024-11-26T23:27:12,809 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=28.3 K, sizeToCheck=16.0 K 2024-11-26T23:27:12,809 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:27:12,809 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/d089b6acdd2641cd9ef6a0065dda35bc because midkey is the same as first or last row 2024-11-26T23:27:12,809 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:27:12,810 DEBUG [RS:0;c707e9ace675:40529-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6644eb6ad870b0b346267b71ec1a8f5b:info 2024-11-26T23:27:12,919 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.1732663616970 to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs/c707e9ace675%2C40529%2C1732663595141.1732663616970 2024-11-26T23:27:12,987 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(556): LowReplication-Roller was enabled. 2024-11-26T23:27:12,988 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.FSHLog(580): java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,148 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T23:27:13,148 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:27:13,148 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:27:13,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:13,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:13,149 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T23:27:13,148 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:27:13,149 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1403695941, stopped=false 2024-11-26T23:27:13,149 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c707e9ace675,33077,1732663595034 2024-11-26T23:27:13,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:27:13,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:27:13,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37097-0x101569a3a720002, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:27:13,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:13,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:13,151 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37097-0x101569a3a720002, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:13,151 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:27:13,151 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:27:13,151 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:27:13,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:13,152 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:27:13,152 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,40529,1732663595141' ***** 2024-11-26T23:27:13,152 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T23:27:13,152 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,37097,1732663596093' ***** 2024-11-26T23:27:13,152 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T23:27:13,152 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:37097-0x101569a3a720002, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:27:13,152 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:27:13,152 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:27:13,152 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:27:13,152 INFO [RS:1;c707e9ace675:37097 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:27:13,152 INFO [RS:0;c707e9ace675:40529 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T23:27:13,152 INFO [RS:1;c707e9ace675:37097 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T23:27:13,152 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:27:13,152 INFO [RS:0;c707e9ace675:40529 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T23:27:13,152 INFO [RS:1;c707e9ace675:37097 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T23:27:13,152 INFO [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer(959): stopping server c707e9ace675,37097,1732663596093 2024-11-26T23:27:13,152 INFO [RS:1;c707e9ace675:37097 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:27:13,153 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(3091): Received CLOSE for 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:27:13,153 INFO [RS:1;c707e9ace675:37097 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c707e9ace675:37097. 2024-11-26T23:27:13,153 DEBUG [RS:1;c707e9ace675:37097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:27:13,153 DEBUG [RS:1;c707e9ace675:37097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:13,153 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T23:27:13,153 INFO [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,37097,1732663596093; all regions closed. 2024-11-26T23:27:13,153 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(959): stopping server c707e9ace675,40529,1732663595141 2024-11-26T23:27:13,153 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T23:27:13,153 INFO [RS:0;c707e9ace675:40529 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:27:13,153 INFO [RS:0;c707e9ace675:40529 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c707e9ace675:40529. 2024-11-26T23:27:13,153 DEBUG [RS:0;c707e9ace675:40529 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:27:13,153 DEBUG [RS:0;c707e9ace675:40529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:13,153 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6644eb6ad870b0b346267b71ec1a8f5b, disabling compactions & flushes 2024-11-26T23:27:13,153 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:27:13,153 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:27:13,153 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. after waiting 0 ms 2024-11-26T23:27:13,153 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:27:13,153 INFO [RS:0;c707e9ace675:40529 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:27:13,154 INFO [RS:0;c707e9ace675:40529 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:27:13,154 INFO [RS:0;c707e9ace675:40529 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:27:13,154 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 6644eb6ad870b0b346267b71ec1a8f5b 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-26T23:27:13,154 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,154 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T23:27:13,154 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,154 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,154 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T23:27:13,154 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,154 DEBUG [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 6644eb6ad870b0b346267b71ec1a8f5b=TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.} 2024-11-26T23:27:13,154 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,154 DEBUG [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 6644eb6ad870b0b346267b71ec1a8f5b 2024-11-26T23:27:13,154 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:27:13,154 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:27:13,154 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:27:13,154 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:27:13,155 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:27:13,155 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.71 KB heapSize=3.75 KB 2024-11-26T23:27:13,155 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,155 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,155 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 2024-11-26T23:27:13,155 ERROR [FSHLog-0-hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad-prefix:c707e9ace675,40529,1732663595141.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,155 WARN [FSHLog-0-hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad-prefix:c707e9ace675,40529,1732663595141.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,155 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C40529%2C1732663595141.meta:.meta(num 1732663595933) roll requested 2024-11-26T23:27:13,156 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C40529%2C1732663595141.meta.1732663633155.meta 2024-11-26T23:27:13,156 WARN [IPC Server handler 4 on default port 34607 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 has not been closed. Lease recovery is in progress. RecoveryId = 1076 for block blk_1073741837_1013 2024-11-26T23:27:13,156 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 after 1ms 2024-11-26T23:27:13,160 WARN [Thread-1052 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741892_1077 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39209 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,160 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:36246 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741892_1077] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data4]'}, localName='127.0.0.1:36419', datanodeUuid='957de82d-0af5-4420-8b0f-1112e576e3dd', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741892_1077 to mirror 127.0.0.1:39209 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:13,160 WARN [Thread-1052 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741892_1077 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK], DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:27:13,160 WARN [Thread-1052 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741892_1077 2024-11-26T23:27:13,160 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/0635ea31099a42d9ba1c4d92055b278a is 1079, key is tmprow/info:/1732663632946/Put/seqid=0 2024-11-26T23:27:13,160 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:36246 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741892_1077] {}] datanode.BlockReceiver(316): Block 1073741892 has not released the reserved bytes. Releasing 268435456 bytes as part of close. 2024-11-26T23:27:13,160 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:36246 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741892_1077] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36246 dst: /127.0.0.1:36419 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:13,161 WARN [Thread-1052 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:27:13,163 WARN [Thread-1053 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741893_1078 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39209 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,163 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:57066 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741893_1078] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8]'}, localName='127.0.0.1:44973', datanodeUuid='c2153758-3f66-4b49-9a42-9c8c6e787d90', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741893_1078 to mirror 127.0.0.1:39209 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:13,163 WARN [Thread-1053 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741893_1078 in pipeline [DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK], DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:27:13,163 WARN [Thread-1053 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741893_1078 2024-11-26T23:27:13,163 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:57066 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741893_1078] {}] datanode.BlockReceiver(316): Block 1073741893 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:27:13,163 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:57066 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741893_1078] {}] datanode.DataXceiver(331): 127.0.0.1:44973:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57066 dst: /127.0.0.1:44973 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:13,164 WARN [Thread-1053 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:27:13,165 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,166 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,166 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,166 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,166 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,166 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta with entries=8, filesize=2.33 KB; new WAL /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663633155.meta 2024-11-26T23:27:13,172 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,172 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:34663,DS-c5a75209-46eb-4bc4-9794-04e9d65dfd35,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,173 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta 2024-11-26T23:27:13,173 WARN [IPC Server handler 0 on default port 34607 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta has not been closed. Lease recovery is in progress. RecoveryId = 1081 for block blk_1073741834_1010 2024-11-26T23:27:13,173 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta after 0ms 2024-11-26T23:27:13,174 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:33751:33751),(127.0.0.1/127.0.0.1:34309:34309)] 2024-11-26T23:27:13,174 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta is not closed yet, will try archiving it next time 2024-11-26T23:27:13,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741895_1080 (size=6027) 2024-11-26T23:27:13,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741895_1080 (size=6027) 2024-11-26T23:27:13,175 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/0635ea31099a42d9ba1c4d92055b278a 2024-11-26T23:27:13,183 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/.tmp/info/0635ea31099a42d9ba1c4d92055b278a as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/0635ea31099a42d9ba1c4d92055b278a 2024-11-26T23:27:13,189 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/0635ea31099a42d9ba1c4d92055b278a, entries=1, sequenceid=84, filesize=5.9 K 2024-11-26T23:27:13,191 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 6644eb6ad870b0b346267b71ec1a8f5b in 36ms, sequenceid=84, compaction requested=false 2024-11-26T23:27:13,191 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/d47c68717e064c25a69eb8593329bf5f, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/9cfb82063a5c4b3d90fc301693ce1d4f, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/64272a46662d4b5a9434f8c621c8db28, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/7a41735b06364bc7a435a31c1bee62c0, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/ccd2a739c89c4078bf29d3e694ff4f12, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/87fc198dd8394089b21381d4038378f5, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/b35d319b80f140578c9e6752aab450fe, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/6d0ff4c365de4853b2016846606f547f, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/22aa6f4cd4ac41a4883573932f41190a] to archive 2024-11-26T23:27:13,192 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:27:13,194 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/d47c68717e064c25a69eb8593329bf5f to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/d47c68717e064c25a69eb8593329bf5f 2024-11-26T23:27:13,196 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/9cfb82063a5c4b3d90fc301693ce1d4f to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/9cfb82063a5c4b3d90fc301693ce1d4f 2024-11-26T23:27:13,197 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/64272a46662d4b5a9434f8c621c8db28 to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/64272a46662d4b5a9434f8c621c8db28 2024-11-26T23:27:13,198 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/.tmp/info/9bae435e58f14a10974183d2c0a5ca22 is 203, key is TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b./info:regioninfo/1732663596554/Put/seqid=0 2024-11-26T23:27:13,199 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/7a41735b06364bc7a435a31c1bee62c0 to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/7a41735b06364bc7a435a31c1bee62c0 2024-11-26T23:27:13,199 WARN [Thread-1066 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741896_1082 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,200 WARN [Thread-1066 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741896_1082 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK], DatanodeInfoWithStorage[127.0.0.1:44973,DS-649476ef-55b2-41c2-b8a9-b56007473276,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:27:13,200 WARN [Thread-1066 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741896_1082 2024-11-26T23:27:13,200 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/ccd2a739c89c4078bf29d3e694ff4f12 to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/ccd2a739c89c4078bf29d3e694ff4f12 2024-11-26T23:27:13,200 WARN [Thread-1066 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:27:13,202 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/87fc198dd8394089b21381d4038378f5 to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/87fc198dd8394089b21381d4038378f5 2024-11-26T23:27:13,203 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/b35d319b80f140578c9e6752aab450fe to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/b35d319b80f140578c9e6752aab450fe 2024-11-26T23:27:13,205 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/6d0ff4c365de4853b2016846606f547f to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/6d0ff4c365de4853b2016846606f547f 2024-11-26T23:27:13,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741897_1083 (size=7089) 2024-11-26T23:27:13,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741897_1083 (size=7089) 2024-11-26T23:27:13,207 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.50 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/.tmp/info/9bae435e58f14a10974183d2c0a5ca22 2024-11-26T23:27:13,207 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/22aa6f4cd4ac41a4883573932f41190a to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/archive/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/info/22aa6f4cd4ac41a4883573932f41190a 2024-11-26T23:27:13,208 DEBUG [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c707e9ace675:33077 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-26T23:27:13,208 WARN [StoreCloser-TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [d47c68717e064c25a69eb8593329bf5f=10347, 9cfb82063a5c4b3d90fc301693ce1d4f=12506, 64272a46662d4b5a9434f8c621c8db28=17994, 7a41735b06364bc7a435a31c1bee62c0=6027, ccd2a739c89c4078bf29d3e694ff4f12=6027, 87fc198dd8394089b21381d4038378f5=18097, b35d319b80f140578c9e6752aab450fe=6027, 6d0ff4c365de4853b2016846606f547f=8190, 22aa6f4cd4ac41a4883573932f41190a=14660] 2024-11-26T23:27:13,213 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/default/TestLogRolling-testLogRollOnDatanodeDeath/6644eb6ad870b0b346267b71ec1a8f5b/recovered.edits/87.seqid, newMaxSeqId=87, maxSeqId=1 2024-11-26T23:27:13,213 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:27:13,214 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6644eb6ad870b0b346267b71ec1a8f5b: Waiting for close lock at 1732663633153Running coprocessor pre-close hooks at 1732663633153Disabling compacts and flushes for region at 1732663633153Disabling writes for close at 1732663633153Obtaining lock to block concurrent updates at 1732663633154 (+1 ms)Preparing flush snapshotting stores in 6644eb6ad870b0b346267b71ec1a8f5b at 1732663633154Finished memstore snapshotting TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b., syncing WAL and waiting on mvcc, flushsize=dataSize=1075, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732663633154Flushing stores of TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. at 1732663633155 (+1 ms)Flushing 6644eb6ad870b0b346267b71ec1a8f5b/info: creating writer at 1732663633155Flushing 6644eb6ad870b0b346267b71ec1a8f5b/info: appending metadata at 1732663633159 (+4 ms)Flushing 6644eb6ad870b0b346267b71ec1a8f5b/info: closing flushed file at 1732663633159Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ef5eee2: reopening flushed file at 1732663633182 (+23 ms)Finished flush of dataSize ~1.05 KB/1075, heapSize ~1.36 KB/1392, currentSize=0 B/0 for 6644eb6ad870b0b346267b71ec1a8f5b in 36ms, sequenceid=84, compaction requested=false at 1732663633191 (+9 ms)Writing region close event to WAL at 1732663633209 (+18 ms)Running coprocessor post-close hooks at 1732663633213 (+4 ms)Closed at 1732663633213 2024-11-26T23:27:13,214 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnDatanodeDeath,,1732663596189.6644eb6ad870b0b346267b71ec1a8f5b. 2024-11-26T23:27:13,237 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/.tmp/ns/f57546e20cfa4381b321f60c0c596dc6 is 43, key is default/ns:d/1732663595975/Put/seqid=0 2024-11-26T23:27:13,241 WARN [Thread-1073 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741898_1084 java.io.IOException: Got error, status=ERROR, status message , ack with firstBadLink as 127.0.0.1:39209 at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:128) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil.checkBlockOpStatus(DataTransferProtoUtil.java:104) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1947) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,241 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:36284 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741898_1084] {}] datanode.DataXceiver(892): DataNode{data=FSDataset{dirpath='[/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data3, /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data4]'}, localName='127.0.0.1:36419', datanodeUuid='957de82d-0af5-4420-8b0f-1112e576e3dd', xmitsInProgress=0}:Exception transferring block BP-1916363482-172.17.0.2-1732663594093:blk_1073741898_1084 to mirror 127.0.0.1:39209 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:13,241 WARN [Thread-1073 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741898_1084 in pipeline [DatanodeInfoWithStorage[127.0.0.1:36419,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK], DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:27:13,241 WARN [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:36284 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741898_1084] {}] datanode.BlockReceiver(316): Block 1073741898 has not released the reserved bytes. Releasing 134217728 bytes as part of close. 2024-11-26T23:27:13,241 WARN [Thread-1073 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741898_1084 2024-11-26T23:27:13,241 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-2123472984_22 at /127.0.0.1:36284 [Receiving block BP-1916363482-172.17.0.2-1732663594093:blk_1073741898_1084] {}] datanode.DataXceiver(331): 127.0.0.1:36419:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36284 dst: /127.0.0.1:36419 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:807) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:13,242 WARN [Thread-1073 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:27:13,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741899_1085 (size=5153) 2024-11-26T23:27:13,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741899_1085 (size=5153) 2024-11-26T23:27:13,249 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/.tmp/ns/f57546e20cfa4381b321f60c0c596dc6 2024-11-26T23:27:13,271 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/.tmp/table/d884e8fa34e243b2b6a945f0a76c7d7a is 77, key is TestLogRolling-testLogRollOnDatanodeDeath/table:state/1732663596564/Put/seqid=0 2024-11-26T23:27:13,272 WARN [Thread-1080 {}] hdfs.DataStreamer(1959): Exception in createBlockOutputStream blk_1073741900_1086 java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createSocketForPipeline(DataStreamer.java:256) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.createBlockOutputStream(DataStreamer.java:1894) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForCreate(DataStreamer.java:1842) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:752) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:13,273 WARN [Thread-1080 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1916363482-172.17.0.2-1732663594093:blk_1073741900_1086 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK], DatanodeInfoWithStorage[127.0.0.1:36419,DS-cacf0762-8e58-4aa7-98b9-88feb8cde337,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK]) is bad. 2024-11-26T23:27:13,273 WARN [Thread-1080 {}] hdfs.DataStreamer(1850): Abandoning BP-1916363482-172.17.0.2-1732663594093:blk_1073741900_1086 2024-11-26T23:27:13,273 WARN [Thread-1080 {}] hdfs.DataStreamer(1857): Excluding datanode DatanodeInfoWithStorage[127.0.0.1:39209,DS-23c99cab-962d-4c98-8c6a-f7396a16f4d4,DISK] 2024-11-26T23:27:13,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741901_1087 (size=5424) 2024-11-26T23:27:13,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741901_1087 (size=5424) 2024-11-26T23:27:13,279 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=146 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/.tmp/table/d884e8fa34e243b2b6a945f0a76c7d7a 2024-11-26T23:27:13,285 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/.tmp/info/9bae435e58f14a10974183d2c0a5ca22 as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/info/9bae435e58f14a10974183d2c0a5ca22 2024-11-26T23:27:13,290 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/info/9bae435e58f14a10974183d2c0a5ca22, entries=10, sequenceid=11, filesize=6.9 K 2024-11-26T23:27:13,291 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/.tmp/ns/f57546e20cfa4381b321f60c0c596dc6 as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/ns/f57546e20cfa4381b321f60c0c596dc6 2024-11-26T23:27:13,296 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/ns/f57546e20cfa4381b321f60c0c596dc6, entries=2, sequenceid=11, filesize=5.0 K 2024-11-26T23:27:13,297 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/.tmp/table/d884e8fa34e243b2b6a945f0a76c7d7a as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/table/d884e8fa34e243b2b6a945f0a76c7d7a 2024-11-26T23:27:13,303 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/table/d884e8fa34e243b2b6a945f0a76c7d7a, entries=2, sequenceid=11, filesize=5.3 K 2024-11-26T23:27:13,304 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 149ms, sequenceid=11, compaction requested=false 2024-11-26T23:27:13,310 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-26T23:27:13,311 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:27:13,311 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:27:13,311 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663633154Running coprocessor pre-close hooks at 1732663633154Disabling compacts and flushes for region at 1732663633154Disabling writes for close at 1732663633155 (+1 ms)Obtaining lock to block concurrent updates at 1732663633155Preparing flush snapshotting stores in 1588230740 at 1732663633155Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1752, getHeapSize=3776, getOffHeapSize=0, getCellsCount=14 at 1732663633155Flushing stores of hbase:meta,,1.1588230740 at 1732663633174 (+19 ms)Flushing 1588230740/info: creating writer at 1732663633174Flushing 1588230740/info: appending metadata at 1732663633197 (+23 ms)Flushing 1588230740/info: closing flushed file at 1732663633197Flushing 1588230740/ns: creating writer at 1732663633214 (+17 ms)Flushing 1588230740/ns: appending metadata at 1732663633237 (+23 ms)Flushing 1588230740/ns: closing flushed file at 1732663633237Flushing 1588230740/table: creating writer at 1732663633255 (+18 ms)Flushing 1588230740/table: appending metadata at 1732663633270 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732663633270Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2660ed78: reopening flushed file at 1732663633284 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1facd576: reopening flushed file at 1732663633290 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29e52eb5: reopening flushed file at 1732663633297 (+7 ms)Finished flush of dataSize ~1.71 KB/1752, heapSize ~3.45 KB/3536, currentSize=0 B/0 for 1588230740 in 149ms, sequenceid=11, compaction requested=false at 1732663633304 (+7 ms)Writing region close event to WAL at 1732663633307 (+3 ms)Running coprocessor post-close hooks at 1732663633311 (+4 ms)Closed at 1732663633311 2024-11-26T23:27:13,312 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T23:27:13,354 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,40529,1732663595141; all regions closed. 2024-11-26T23:27:13,355 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,355 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,355 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,355 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,356 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:13,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741894_1079 (size=825) 2024-11-26T23:27:13,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741894_1079 (size=825) 2024-11-26T23:27:13,444 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T23:27:13,444 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T23:27:13,447 INFO [regionserver/c707e9ace675:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:27:13,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741874_1057 (size=12911) 2024-11-26T23:27:14,150 INFO [regionserver/c707e9ace675:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:27:14,292 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@1f546071[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=957de82d-0af5-4420-8b0f-1112e576e3dd, infoPort=33751, infoSecurePort=0, ipcPort=37035, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741836_1012 to 127.0.0.1:39209 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:14,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:27:15,292 WARN [java.util.concurrent.ThreadPoolExecutor$Worker@1f546071[State = -1, empty queue] {}] datanode.DataNode$DataTransfer(3129): DatanodeRegistration(127.0.0.1:36419, datanodeUuid=957de82d-0af5-4420-8b0f-1112e576e3dd, infoPort=33751, infoSecurePort=0, ipcPort=37035, storageInfo=lv=-57;cid=testClusterID;nsid=437933727;c=1732663594093):Failed to transfer BP-1916363482-172.17.0.2-1732663594093:blk_1073741826_1002 to 127.0.0.1:39209 got java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:577) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataNode$DataTransfer.run(DataNode.java:3063) ~[hadoop-hdfs-3.4.1.jar:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:15,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:27:15,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnDatanodeDeath 2024-11-26T23:27:15,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:27:15,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T23:27:15,991 INFO [master/c707e9ace675:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T23:27:15,991 INFO [master/c707e9ace675:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T23:27:17,157 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 after 4002ms 2024-11-26T23:27:17,174 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=1 on file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta after 4001ms 2024-11-26T23:27:17,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:27:17,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:27:18,155 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-26T23:27:18,157 DEBUG [RS:1;c707e9ace675:37097 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs 2024-11-26T23:27:18,157 INFO [RS:1;c707e9ace675:37097 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C37097%2C1732663596093:(num 1732663596289) 2024-11-26T23:27:18,157 DEBUG [RS:1;c707e9ace675:37097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:18,157 INFO [RS:1;c707e9ace675:37097 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:27:18,157 INFO [RS:1;c707e9ace675:37097 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:27:18,158 INFO [RS:1;c707e9ace675:37097 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-26T23:27:18,158 INFO [RS:1;c707e9ace675:37097 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:27:18,158 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:27:18,158 INFO [RS:1;c707e9ace675:37097 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:27:18,158 INFO [RS:1;c707e9ace675:37097 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:27:18,158 INFO [RS:1;c707e9ace675:37097 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:27:18,158 INFO [RS:1;c707e9ace675:37097 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37097 2024-11-26T23:27:18,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:27:18,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37097-0x101569a3a720002, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,37097,1732663596093 2024-11-26T23:27:18,160 INFO [RS:1;c707e9ace675:37097 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:27:18,162 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,37097,1732663596093] 2024-11-26T23:27:18,163 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,37097,1732663596093 already deleted, retry=false 2024-11-26T23:27:18,163 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c707e9ace675,37097,1732663596093 expired; onlineServers=1 2024-11-26T23:27:18,163 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.FileNotFoundException: File does not exist: /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:88) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1812) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:87) at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:77) at org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.isFileClosed(FSDirStatAndListingOp.java:124) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.isFileClosed(FSNamesystem.java:3502) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.isFileClosed(NameNodeRpcServer.java:1248) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.isFileClosed(ClientNamenodeProtocolServerSideTranslatorPB.java:1419) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$isFileClosed$57(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.isFileClosed(ClientNamenodeProtocolTranslatorPB.java:999) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor104.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.isFileClosed(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1810) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:18,214 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,228 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,228 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,229 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,229 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,229 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,236 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,237 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37097-0x101569a3a720002, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:27:18,262 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:37097-0x101569a3a720002, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:27:18,262 INFO [RS:1;c707e9ace675:37097 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:27:18,262 INFO [RS:1;c707e9ace675:37097 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,37097,1732663596093; zookeeper connection closed. 2024-11-26T23:27:18,262 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6d48aa28 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6d48aa28 2024-11-26T23:27:18,356 ERROR [WAL-Shutdown-0 {}] wal.AbstractFSWAL(2118): We have waited 5 seconds but the close of async writer doesn't complete.Please check the status of underlying filesystem or increase the wait time by the config "hbase.wal.fshlog.wait.on.shutdown.seconds" 2024-11-26T23:27:18,359 DEBUG [RS:0;c707e9ace675:40529 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs 2024-11-26T23:27:18,359 INFO [RS:0;c707e9ace675:40529 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C40529%2C1732663595141.meta:.meta(num 1732663633155) 2024-11-26T23:27:18,360 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:18,360 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:18,360 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:18,360 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:18,360 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:18,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741886_1070 (size=18156) 2024-11-26T23:27:18,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741886_1070 (size=18156) 2024-11-26T23:27:18,364 DEBUG [RS:0;c707e9ace675:40529 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs 2024-11-26T23:27:18,364 INFO [RS:0;c707e9ace675:40529 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C40529%2C1732663595141:(num 1732663632510) 2024-11-26T23:27:18,364 DEBUG [RS:0;c707e9ace675:40529 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:18,364 INFO [RS:0;c707e9ace675:40529 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:27:18,364 INFO [RS:0;c707e9ace675:40529 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:27:18,365 INFO [RS:0;c707e9ace675:40529 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T23:27:18,365 INFO [RS:0;c707e9ace675:40529 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:27:18,365 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:27:18,365 INFO [RS:0;c707e9ace675:40529 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40529 2024-11-26T23:27:18,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,40529,1732663595141 2024-11-26T23:27:18,369 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:27:18,369 INFO [RS:0;c707e9ace675:40529 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:27:18,371 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,40529,1732663595141] 2024-11-26T23:27:18,372 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,40529,1732663595141 already deleted, retry=false 2024-11-26T23:27:18,372 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c707e9ace675,40529,1732663595141 expired; onlineServers=0 2024-11-26T23:27:18,372 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c707e9ace675,33077,1732663595034' ***** 2024-11-26T23:27:18,372 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T23:27:18,372 INFO [M:0;c707e9ace675:33077 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:27:18,372 INFO [M:0;c707e9ace675:33077 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:27:18,372 DEBUG [M:0;c707e9ace675:33077 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T23:27:18,372 DEBUG [M:0;c707e9ace675:33077 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T23:27:18,372 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T23:27:18,372 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663595324 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663595324,5,FailOnTimeoutGroup] 2024-11-26T23:27:18,372 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663595325 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663595325,5,FailOnTimeoutGroup] 2024-11-26T23:27:18,373 INFO [M:0;c707e9ace675:33077 {}] hbase.ChoreService(370): Chore service for: master/c707e9ace675:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T23:27:18,373 INFO [M:0;c707e9ace675:33077 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:27:18,373 DEBUG [M:0;c707e9ace675:33077 {}] master.HMaster(1795): Stopping service threads 2024-11-26T23:27:18,373 INFO [M:0;c707e9ace675:33077 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T23:27:18,373 INFO [M:0;c707e9ace675:33077 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:27:18,373 INFO [M:0;c707e9ace675:33077 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T23:27:18,373 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T23:27:18,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T23:27:18,374 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:18,374 DEBUG [M:0;c707e9ace675:33077 {}] zookeeper.ZKUtil(347): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T23:27:18,374 WARN [M:0;c707e9ace675:33077 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T23:27:18,375 INFO [M:0;c707e9ace675:33077 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/.lastflushedseqids 2024-11-26T23:27:18,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741902_1088 (size=130) 2024-11-26T23:27:18,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741902_1088 (size=130) 2024-11-26T23:27:18,381 INFO [M:0;c707e9ace675:33077 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T23:27:18,382 INFO [M:0;c707e9ace675:33077 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T23:27:18,382 DEBUG [M:0;c707e9ace675:33077 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:27:18,382 INFO [M:0;c707e9ace675:33077 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:18,382 DEBUG [M:0;c707e9ace675:33077 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:18,382 DEBUG [M:0;c707e9ace675:33077 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:27:18,382 DEBUG [M:0;c707e9ace675:33077 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:18,382 INFO [M:0;c707e9ace675:33077 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.25 KB heapSize=29.49 KB 2024-11-26T23:27:18,398 DEBUG [M:0;c707e9ace675:33077 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5a0cbd6997da4326b118a94d72551b46 is 82, key is hbase:meta,,1/info:regioninfo/1732663595959/Put/seqid=0 2024-11-26T23:27:18,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741903_1089 (size=5672) 2024-11-26T23:27:18,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741903_1089 (size=5672) 2024-11-26T23:27:18,403 INFO [M:0;c707e9ace675:33077 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5a0cbd6997da4326b118a94d72551b46 2024-11-26T23:27:18,423 DEBUG [M:0;c707e9ace675:33077 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77ba9e2fb32e40519e49a6561792ae11 is 774, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732663596569/Put/seqid=0 2024-11-26T23:27:18,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741904_1090 (size=6255) 2024-11-26T23:27:18,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741904_1090 (size=6255) 2024-11-26T23:27:18,429 INFO [M:0;c707e9ace675:33077 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.58 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77ba9e2fb32e40519e49a6561792ae11 2024-11-26T23:27:18,434 INFO [M:0;c707e9ace675:33077 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 77ba9e2fb32e40519e49a6561792ae11 2024-11-26T23:27:18,448 DEBUG [M:0;c707e9ace675:33077 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3df90a3c129f4b28b3d841f5d36df710 is 69, key is c707e9ace675,37097,1732663596093/rs:state/1732663596135/Put/seqid=0 2024-11-26T23:27:18,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741905_1091 (size=5224) 2024-11-26T23:27:18,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741905_1091 (size=5224) 2024-11-26T23:27:18,453 INFO [M:0;c707e9ace675:33077 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=130 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3df90a3c129f4b28b3d841f5d36df710 2024-11-26T23:27:18,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:27:18,471 INFO [RS:0;c707e9ace675:40529 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:27:18,471 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40529-0x101569a3a720001, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:27:18,471 INFO [RS:0;c707e9ace675:40529 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,40529,1732663595141; zookeeper connection closed. 2024-11-26T23:27:18,471 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@78a6e08b {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@78a6e08b 2024-11-26T23:27:18,471 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 2 regionserver(s) complete 2024-11-26T23:27:18,472 DEBUG [M:0;c707e9ace675:33077 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ee3d6a47f7f246ee911ec1866605ea07 is 52, key is load_balancer_on/state:d/1732663596075/Put/seqid=0 2024-11-26T23:27:18,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741906_1092 (size=5056) 2024-11-26T23:27:18,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741906_1092 (size=5056) 2024-11-26T23:27:18,477 INFO [M:0;c707e9ace675:33077 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ee3d6a47f7f246ee911ec1866605ea07 2024-11-26T23:27:18,482 DEBUG [M:0;c707e9ace675:33077 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/5a0cbd6997da4326b118a94d72551b46 as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5a0cbd6997da4326b118a94d72551b46 2024-11-26T23:27:18,487 INFO [M:0;c707e9ace675:33077 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/5a0cbd6997da4326b118a94d72551b46, entries=8, sequenceid=60, filesize=5.5 K 2024-11-26T23:27:18,488 DEBUG [M:0;c707e9ace675:33077 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/77ba9e2fb32e40519e49a6561792ae11 as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/77ba9e2fb32e40519e49a6561792ae11 2024-11-26T23:27:18,492 INFO [M:0;c707e9ace675:33077 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 77ba9e2fb32e40519e49a6561792ae11 2024-11-26T23:27:18,492 INFO [M:0;c707e9ace675:33077 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/77ba9e2fb32e40519e49a6561792ae11, entries=6, sequenceid=60, filesize=6.1 K 2024-11-26T23:27:18,493 DEBUG [M:0;c707e9ace675:33077 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3df90a3c129f4b28b3d841f5d36df710 as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3df90a3c129f4b28b3d841f5d36df710 2024-11-26T23:27:18,498 INFO [M:0;c707e9ace675:33077 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3df90a3c129f4b28b3d841f5d36df710, entries=2, sequenceid=60, filesize=5.1 K 2024-11-26T23:27:18,498 DEBUG [M:0;c707e9ace675:33077 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ee3d6a47f7f246ee911ec1866605ea07 as hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ee3d6a47f7f246ee911ec1866605ea07 2024-11-26T23:27:18,503 INFO [M:0;c707e9ace675:33077 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ee3d6a47f7f246ee911ec1866605ea07, entries=1, sequenceid=60, filesize=4.9 K 2024-11-26T23:27:18,504 INFO [M:0;c707e9ace675:33077 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 122ms, sequenceid=60, compaction requested=false 2024-11-26T23:27:18,505 INFO [M:0;c707e9ace675:33077 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:18,505 DEBUG [M:0;c707e9ace675:33077 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663638382Disabling compacts and flushes for region at 1732663638382Disabling writes for close at 1732663638382Obtaining lock to block concurrent updates at 1732663638382Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732663638382Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23805, getHeapSize=30136, getOffHeapSize=0, getCellsCount=71 at 1732663638382Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732663638383 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732663638383Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732663638398 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732663638398Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732663638409 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732663638423 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732663638423Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732663638434 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732663638448 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732663638448Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732663638458 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732663638472 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732663638472Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7dfb41f6: reopening flushed file at 1732663638481 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33fc4ce4: reopening flushed file at 1732663638487 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e36b2e6: reopening flushed file at 1732663638492 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66638bdf: reopening flushed file at 1732663638498 (+6 ms)Finished flush of dataSize ~23.25 KB/23805, heapSize ~29.43 KB/30136, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 122ms, sequenceid=60, compaction requested=false at 1732663638504 (+6 ms)Writing region close event to WAL at 1732663638505 (+1 ms)Closed at 1732663638505 2024-11-26T23:27:18,506 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:18,506 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:18,506 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:18,506 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:18,506 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:18,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44973 is added to blk_1073741885_1068 (size=1045) 2024-11-26T23:27:18,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36419 is added to blk_1073741885_1068 (size=1045) 2024-11-26T23:27:18,739 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-26T23:27:18,754 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,754 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,755 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,755 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,755 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,756 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,761 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:18,765 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:19,164 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:19,175 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:19,319 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@67d37931 {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1916363482-172.17.0.2-1732663594093:blk_1073741830_1006, datanode=DatanodeInfoWithStorage[127.0.0.1:34663,null,null]) java.net.ConnectException: Call From c707e9ace675/172.17.0.2 to localhost:44355 failed on connection exception: java.net.ConnectException: Connection refused; For more details see: http://wiki.apache.org/hadoop/ConnectionRefused at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:876) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout.connect(SocketIOWithTimeout.java:205) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.connect(NetUtils.java:614) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:668) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-26T23:27:19,338 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/WALs/c707e9ace675,33077,1732663595034/c707e9ace675%2C33077%2C1732663595034.1732663595240 to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/oldWALs/c707e9ace675%2C33077%2C1732663595034.1732663595240 2024-11-26T23:27:19,341 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/MasterData/oldWALs/c707e9ace675%2C33077%2C1732663595034.1732663595240 to hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/oldWALs/c707e9ace675%2C33077%2C1732663595034.1732663595240$masterlocalwal$ 2024-11-26T23:27:19,341 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:27:19,341 INFO [M:0;c707e9ace675:33077 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T23:27:19,342 INFO [M:0;c707e9ace675:33077 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33077 2024-11-26T23:27:19,342 INFO [M:0;c707e9ace675:33077 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:27:19,445 INFO [M:0;c707e9ace675:33077 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:27:19,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:27:19,445 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33077-0x101569a3a720000, quorum=127.0.0.1:53009, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:27:19,447 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d95479b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:19,447 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@469c1f5d{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:27:19,447 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:27:19,448 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1191c470{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:27:19,448 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@13c2f5a4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,STOPPED} 2024-11-26T23:27:19,449 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:27:19,449 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:27:19,449 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:27:19,449 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1916363482-172.17.0.2-1732663594093 (Datanode Uuid 957de82d-0af5-4420-8b0f-1112e576e3dd) service to localhost/127.0.0.1:34607 2024-11-26T23:27:19,449 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@47300b2b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:34663,null,null]) java.io.InterruptedIOException: DestHost:destPort localhost:44355 , LocalHost:localPort c707e9ace675/172.17.0.2:0. Failed on local exception: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.net.NetUtils.wrapWithMessage(NetUtils.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.NetUtils.wrapException(NetUtils.java:936) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1588) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy55.initReplicaRecovery(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.lambda$initReplicaRecovery$0(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.InterDatanodeProtocolTranslatorPB.initReplicaRecovery(InterDatanodeProtocolTranslatorPB.java:82) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.callInitReplicaRecovery(BlockRecoveryWorker.java:561) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$400(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:135) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.InterruptedIOException: Interrupted: action=RetryAction(action=RETRY, delayMillis=1000, reason=retries get failed due to exceeded maximum allowed retries number: 10), retry policy=RetryUpToMaximumCountWithFixedSleep(maxRetries=10, sleepTime=1000 MILLISECONDS) at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:963) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more Caused by: java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.ipc.Client$Connection.handleConnectionFailure(Client.java:961) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupConnection(Client.java:691) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.setupIOstreams(Client.java:789) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client$Connection.access$3800(Client.java:364) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.getConnection(Client.java:1649) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1473) ~[hadoop-common-3.4.1.jar:?] ... 12 more 2024-11-26T23:27:19,450 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@47300b2b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013, datanode=DatanodeInfoWithStorage[127.0.0.1:36419,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1916363482-172.17.0.2-1732663594093 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:19,450 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data3/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:19,450 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@47300b2b {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:34663,null,null], DatanodeInfoWithStorage[127.0.0.1:36419,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1916363482-172.17.0.2-1732663594093:blk_1073741837_1013, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:34663,null,null], DatanodeInfoWithStorage[127.0.0.1:36419,null,null]] 2024-11-26T23:27:19,450 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@47300b2b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:34663,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1916363482-172.17.0.2-1732663594093 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:19,450 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@47300b2b {}] datanode.BlockRecoveryWorker$RecoveryTaskContiguous(164): Failed to recover block (block=BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010, datanode=DatanodeInfoWithStorage[127.0.0.1:36419,null,null]) java.io.IOException: No block pool offer service for bpid=BP-1916363482-172.17.0.2-1732663594093 at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.getDatanodeID(BlockRecoveryWorker.java:539) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker.access$000(BlockRecoveryWorker.java:57) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$RecoveryTaskContiguous.recover(BlockRecoveryWorker.java:131) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1.run(BlockRecoveryWorker.java:602) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:19,450 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data4/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:19,450 WARN [org.apache.hadoop.hdfs.server.datanode.BlockRecoveryWorker$1@47300b2b {}] datanode.BlockRecoveryWorker$1(605): recover Block: RecoveringBlock{BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010; getBlockSize()=85; corrupt=false; offset=-1; locs=[DatanodeInfoWithStorage[127.0.0.1:34663,null,null], DatanodeInfoWithStorage[127.0.0.1:36419,null,null]]; cachedLocs=[]} FAILED: java.io.IOException: All datanodes failed: block=BP-1916363482-172.17.0.2-1732663594093:blk_1073741834_1010, datanodeids=[DatanodeInfoWithStorage[127.0.0.1:34663,null,null], DatanodeInfoWithStorage[127.0.0.1:36419,null,null]] 2024-11-26T23:27:19,450 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:27:19,452 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3df1987c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:19,453 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@71bfb2ba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:27:19,453 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:27:19,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7fbc343d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:27:19,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fb99827{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,STOPPED} 2024-11-26T23:27:19,459 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:27:19,459 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:27:19,459 WARN [BP-1916363482-172.17.0.2-1732663594093 heartbeating to localhost/127.0.0.1:34607 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1916363482-172.17.0.2-1732663594093 (Datanode Uuid c2153758-3f66-4b49-9a42-9c8c6e787d90) service to localhost/127.0.0.1:34607 2024-11-26T23:27:19,459 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:27:19,459 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data7/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:19,460 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/cluster_a1a97d6f-88a1-62c2-22a3-c7a6fe02c8e8/data/data8/current/BP-1916363482-172.17.0.2-1732663594093 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:19,460 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:27:19,466 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5519c514{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:27:19,466 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@47f2ada2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:27:19,466 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:27:19,467 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@35c95cb4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:27:19,467 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@17c48ca{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir/,STOPPED} 2024-11-26T23:27:19,475 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T23:27:19,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T23:27:19,511 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnDatanodeDeath Thread=154 (was 79) Potentially hanging thread: LeaseRenewer:jenkins@localhost:34607 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34607 from jenkins.hfs.3 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-17-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fbea0bf4ff8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34607 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34607 from jenkins.hfs.2 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-21-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.1@localhost:42807 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.2@localhost:34607 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:42807 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34607 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-9-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.3@localhost:34607 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34607 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-21-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-5-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34607 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-21-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34607 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-20-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Close-WAL-Writer-0 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:175) app//org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) app//org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL$$Lambda$900/0x00007fbea0bf4ff8.run(Unknown Source) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-8-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-17-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-6-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-7-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'DataNode' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34607 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-16-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=452 (was 402) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=180 (was 204), ProcessCount=11 (was 11), AvailableMemoryMB=6787 (was 6778) - AvailableMemoryMB LEAK? - 2024-11-26T23:27:19,519 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=154, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=180, ProcessCount=11, AvailableMemoryMB=6787 2024-11-26T23:27:19,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T23:27:19,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.log.dir so I do NOT create it in target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176 2024-11-26T23:27:19,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3acda823-82c4-eaaf-a5f2-cbac9dc0a217/hadoop.tmp.dir so I do NOT create it in target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176 2024-11-26T23:27:19,519 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f, deleteOnExit=true 2024-11-26T23:27:19,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T23:27:19,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/test.cache.data in system properties and HBase conf 2024-11-26T23:27:19,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T23:27:19,519 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir in system properties and HBase conf 2024-11-26T23:27:19,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T23:27:19,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T23:27:19,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T23:27:19,520 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T23:27:19,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:27:19,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:27:19,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T23:27:19,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:27:19,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T23:27:19,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T23:27:19,520 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:27:19,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:27:19,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T23:27:19,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/nfs.dump.dir in system properties and HBase conf 2024-11-26T23:27:19,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/java.io.tmpdir in system properties and HBase conf 2024-11-26T23:27:19,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:27:19,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T23:27:19,521 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T23:27:19,534 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:27:19,593 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:19,597 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:27:19,598 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:27:19,598 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:27:19,599 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:27:19,599 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:19,599 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6495f923{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:27:19,600 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d60493b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:27:19,715 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3e84569b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/java.io.tmpdir/jetty-localhost-38987-hadoop-hdfs-3_4_1-tests_jar-_-any-7114507857870774084/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:27:19,716 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@ea144a0{HTTP/1.1, (http/1.1)}{localhost:38987} 2024-11-26T23:27:19,716 INFO [Time-limited test {}] server.Server(415): Started @149061ms 2024-11-26T23:27:19,729 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:27:19,790 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:19,793 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:27:19,794 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:27:19,794 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:27:19,794 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:27:19,794 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@78e57521{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:27:19,795 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@168478e8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:27:19,909 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7096145a{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/java.io.tmpdir/jetty-localhost-34409-hadoop-hdfs-3_4_1-tests_jar-_-any-12256006726286311187/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:19,910 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13bf27af{HTTP/1.1, (http/1.1)}{localhost:34409} 2024-11-26T23:27:19,910 INFO [Time-limited test {}] server.Server(415): Started @149255ms 2024-11-26T23:27:19,912 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:27:19,940 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:19,943 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:27:19,944 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:27:19,944 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:27:19,944 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:27:19,945 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c6cb23c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:27:19,945 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@365c2477{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:27:20,003 WARN [Thread-1201 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data1/current/BP-1882697377-172.17.0.2-1732663639542/current, will proceed with Du for space computation calculation, 2024-11-26T23:27:20,003 WARN [Thread-1202 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data2/current/BP-1882697377-172.17.0.2-1732663639542/current, will proceed with Du for space computation calculation, 2024-11-26T23:27:20,024 WARN [Thread-1180 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:27:20,027 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8e22f4bf19165161 with lease ID 0xd0e1ecba14555495: Processing first storage report for DS-a2f3e615-428c-4f46-baa4-af09b5c855d3 from datanode DatanodeRegistration(127.0.0.1:33077, datanodeUuid=0c70b437-1d71-46c0-8634-2206e7a6113e, infoPort=37365, infoSecurePort=0, ipcPort=36385, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542) 2024-11-26T23:27:20,027 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8e22f4bf19165161 with lease ID 0xd0e1ecba14555495: from storage DS-a2f3e615-428c-4f46-baa4-af09b5c855d3 node DatanodeRegistration(127.0.0.1:33077, datanodeUuid=0c70b437-1d71-46c0-8634-2206e7a6113e, infoPort=37365, infoSecurePort=0, ipcPort=36385, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:20,027 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8e22f4bf19165161 with lease ID 0xd0e1ecba14555495: Processing first storage report for DS-8fa02e10-e904-4dfe-aeae-fd152ef1dc32 from datanode DatanodeRegistration(127.0.0.1:33077, datanodeUuid=0c70b437-1d71-46c0-8634-2206e7a6113e, infoPort=37365, infoSecurePort=0, ipcPort=36385, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542) 2024-11-26T23:27:20,027 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8e22f4bf19165161 with lease ID 0xd0e1ecba14555495: from storage DS-8fa02e10-e904-4dfe-aeae-fd152ef1dc32 node DatanodeRegistration(127.0.0.1:33077, datanodeUuid=0c70b437-1d71-46c0-8634-2206e7a6113e, infoPort=37365, infoSecurePort=0, ipcPort=36385, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:20,069 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@30a928dc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/java.io.tmpdir/jetty-localhost-39987-hadoop-hdfs-3_4_1-tests_jar-_-any-1942607255073515128/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:20,070 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@341f9f9e{HTTP/1.1, (http/1.1)}{localhost:39987} 2024-11-26T23:27:20,070 INFO [Time-limited test {}] server.Server(415): Started @149415ms 2024-11-26T23:27:20,072 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:27:20,164 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:20,171 WARN [Thread-1227 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data3/current/BP-1882697377-172.17.0.2-1732663639542/current, will proceed with Du for space computation calculation, 2024-11-26T23:27:20,171 WARN [Thread-1228 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data4/current/BP-1882697377-172.17.0.2-1732663639542/current, will proceed with Du for space computation calculation, 2024-11-26T23:27:20,175 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:20,193 WARN [Thread-1216 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:27:20,196 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc72cc61129d424f2 with lease ID 0xd0e1ecba14555496: Processing first storage report for DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c from datanode DatanodeRegistration(127.0.0.1:33325, datanodeUuid=9d9cb261-b1e1-476f-85b2-4c9eadd145e1, infoPort=32841, infoSecurePort=0, ipcPort=33687, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542) 2024-11-26T23:27:20,196 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc72cc61129d424f2 with lease ID 0xd0e1ecba14555496: from storage DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c node DatanodeRegistration(127.0.0.1:33325, datanodeUuid=9d9cb261-b1e1-476f-85b2-4c9eadd145e1, infoPort=32841, infoSecurePort=0, ipcPort=33687, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:20,196 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xc72cc61129d424f2 with lease ID 0xd0e1ecba14555496: Processing first storage report for DS-66e4ef7a-5b25-4bec-980f-55283a7b2e0b from datanode DatanodeRegistration(127.0.0.1:33325, datanodeUuid=9d9cb261-b1e1-476f-85b2-4c9eadd145e1, infoPort=32841, infoSecurePort=0, ipcPort=33687, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542) 2024-11-26T23:27:20,196 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xc72cc61129d424f2 with lease ID 0xd0e1ecba14555496: from storage DS-66e4ef7a-5b25-4bec-980f-55283a7b2e0b node DatanodeRegistration(127.0.0.1:33325, datanodeUuid=9d9cb261-b1e1-476f-85b2-4c9eadd145e1, infoPort=32841, infoSecurePort=0, ipcPort=33687, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:20,297 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176 2024-11-26T23:27:20,300 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/zookeeper_0, clientPort=52559, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T23:27:20,301 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52559 2024-11-26T23:27:20,301 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:20,303 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:20,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33077 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:27:20,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:27:20,312 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c with version=8 2024-11-26T23:27:20,313 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/hbase-staging 2024-11-26T23:27:20,315 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:27:20,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:20,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:20,315 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:27:20,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:20,315 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:27:20,315 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T23:27:20,315 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:27:20,316 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42489 2024-11-26T23:27:20,317 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42489 connecting to ZooKeeper ensemble=127.0.0.1:52559 2024-11-26T23:27:20,323 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:424890x0, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:27:20,323 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42489-0x101569aeb7b0000 connected 2024-11-26T23:27:20,338 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:20,340 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:20,341 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:27:20,342 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c, hbase.cluster.distributed=false 2024-11-26T23:27:20,343 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:27:20,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42489 2024-11-26T23:27:20,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42489 2024-11-26T23:27:20,344 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42489 2024-11-26T23:27:20,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42489 2024-11-26T23:27:20,345 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42489 2024-11-26T23:27:20,360 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:27:20,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:20,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:20,360 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:27:20,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:20,360 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:27:20,360 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:27:20,360 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:27:20,361 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41323 2024-11-26T23:27:20,362 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41323 connecting to ZooKeeper ensemble=127.0.0.1:52559 2024-11-26T23:27:20,363 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:20,365 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:20,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:413230x0, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:27:20,370 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:413230x0, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:27:20,370 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41323-0x101569aeb7b0001 connected 2024-11-26T23:27:20,370 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:27:20,371 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:27:20,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T23:27:20,373 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:27:20,373 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41323 2024-11-26T23:27:20,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41323 2024-11-26T23:27:20,374 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41323 2024-11-26T23:27:20,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41323 2024-11-26T23:27:20,378 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41323 2024-11-26T23:27:20,390 DEBUG [M:0;c707e9ace675:42489 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c707e9ace675:42489 2024-11-26T23:27:20,391 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c707e9ace675,42489,1732663640314 2024-11-26T23:27:20,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:27:20,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:27:20,393 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c707e9ace675,42489,1732663640314 2024-11-26T23:27:20,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T23:27:20,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,397 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T23:27:20,398 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c707e9ace675,42489,1732663640314 from backup master directory 2024-11-26T23:27:20,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c707e9ace675,42489,1732663640314 2024-11-26T23:27:20,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:27:20,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:27:20,399 WARN [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:27:20,399 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c707e9ace675,42489,1732663640314 2024-11-26T23:27:20,403 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/hbase.id] with ID: dcf238fd-6d33-450f-8e64-d44aea44b72c 2024-11-26T23:27:20,403 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/.tmp/hbase.id 2024-11-26T23:27:20,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33077 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:27:20,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:27:20,410 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/.tmp/hbase.id]:[hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/hbase.id] 2024-11-26T23:27:20,421 INFO [master/c707e9ace675:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:20,421 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T23:27:20,422 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-26T23:27:20,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33077 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:27:20,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:27:20,435 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:27:20,436 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T23:27:20,436 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:27:20,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:27:20,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33077 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:27:20,445 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store 2024-11-26T23:27:20,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:27:20,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33077 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:27:20,452 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:20,452 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:27:20,452 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:20,452 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:20,452 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:27:20,452 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:20,453 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:20,453 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663640452Disabling compacts and flushes for region at 1732663640452Disabling writes for close at 1732663640452Writing region close event to WAL at 1732663640452Closed at 1732663640452 2024-11-26T23:27:20,453 WARN [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/.initializing 2024-11-26T23:27:20,453 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314 2024-11-26T23:27:20,456 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C42489%2C1732663640314, suffix=, logDir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314, archiveDir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/oldWALs, maxLogs=10 2024-11-26T23:27:20,456 INFO [master/c707e9ace675:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C42489%2C1732663640314.1732663640456 2024-11-26T23:27:20,461 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314/c707e9ace675%2C42489%2C1732663640314.1732663640456 2024-11-26T23:27:20,466 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37365:37365),(127.0.0.1/127.0.0.1:32841:32841)] 2024-11-26T23:27:20,467 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:27:20,467 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:20,467 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,467 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,469 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,470 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T23:27:20,470 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:20,470 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:20,471 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T23:27:20,472 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:20,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:27:20,472 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,473 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T23:27:20,473 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:20,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:27:20,474 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T23:27:20,475 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:20,475 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:27:20,475 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,476 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,476 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,477 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,477 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,478 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:27:20,479 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:20,481 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:27:20,481 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=826663, jitterRate=0.05115698277950287}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:27:20,482 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732663640467Initializing all the Stores at 1732663640468 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663640468Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663640468Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663640468Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663640468Cleaning up temporary data from old regions at 1732663640477 (+9 ms)Region opened successfully at 1732663640482 (+5 ms) 2024-11-26T23:27:20,482 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T23:27:20,484 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72796aeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:27:20,485 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T23:27:20,485 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T23:27:20,485 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T23:27:20,485 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T23:27:20,486 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T23:27:20,486 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-26T23:27:20,486 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T23:27:20,489 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T23:27:20,490 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T23:27:20,491 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T23:27:20,492 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T23:27:20,492 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T23:27:20,495 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T23:27:20,495 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T23:27:20,496 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T23:27:20,497 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T23:27:20,498 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T23:27:20,499 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T23:27:20,501 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T23:27:20,502 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T23:27:20,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:27:20,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:27:20,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,505 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,505 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c707e9ace675,42489,1732663640314, sessionid=0x101569aeb7b0000, setting cluster-up flag (Was=false) 2024-11-26T23:27:20,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,508 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,516 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T23:27:20,517 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,42489,1732663640314 2024-11-26T23:27:20,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,521 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,527 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T23:27:20,528 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,42489,1732663640314 2024-11-26T23:27:20,530 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T23:27:20,532 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T23:27:20,532 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T23:27:20,532 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T23:27:20,533 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c707e9ace675,42489,1732663640314 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T23:27:20,534 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:27:20,534 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:27:20,534 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:27:20,534 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:27:20,534 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c707e9ace675:0, corePoolSize=10, maxPoolSize=10 2024-11-26T23:27:20,534 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,534 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:27:20,535 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,537 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:27:20,537 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T23:27:20,538 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:20,538 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732663670538 2024-11-26T23:27:20,538 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T23:27:20,538 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T23:27:20,539 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T23:27:20,539 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T23:27:20,539 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T23:27:20,539 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T23:27:20,539 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T23:27:20,539 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,539 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T23:27:20,539 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T23:27:20,539 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T23:27:20,540 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T23:27:20,540 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T23:27:20,540 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663640540,5,FailOnTimeoutGroup] 2024-11-26T23:27:20,544 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663640540,5,FailOnTimeoutGroup] 2024-11-26T23:27:20,544 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,544 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T23:27:20,544 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,544 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33077 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:27:20,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:27:20,550 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T23:27:20,550 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c 2024-11-26T23:27:20,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:27:20,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33077 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:27:20,559 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:20,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:27:20,562 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:27:20,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:20,562 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:20,563 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:27:20,564 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:27:20,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:20,564 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:20,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:27:20,566 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:27:20,566 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:20,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:20,566 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:27:20,567 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:27:20,567 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:20,568 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:20,568 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:27:20,569 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740 2024-11-26T23:27:20,569 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740 2024-11-26T23:27:20,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:27:20,570 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:27:20,571 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:27:20,572 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:27:20,574 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:27:20,574 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=792190, jitterRate=0.007322847843170166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:27:20,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732663640559Initializing all the Stores at 1732663640560 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663640560Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663640560Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663640560Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663640560Cleaning up temporary data from old regions at 1732663640570 (+10 ms)Region opened successfully at 1732663640575 (+5 ms) 2024-11-26T23:27:20,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:27:20,575 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:27:20,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:27:20,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:27:20,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:27:20,576 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:27:20,576 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663640575Disabling compacts and flushes for region at 1732663640575Disabling writes for close at 1732663640575Writing region close event to WAL at 1732663640575Closed at 1732663640575 2024-11-26T23:27:20,577 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:27:20,577 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T23:27:20,577 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T23:27:20,578 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:27:20,580 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T23:27:20,582 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(746): ClusterId : dcf238fd-6d33-450f-8e64-d44aea44b72c 2024-11-26T23:27:20,582 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:27:20,584 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:27:20,584 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:27:20,587 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:27:20,587 DEBUG [RS:0;c707e9ace675:41323 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634c11af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:27:20,599 DEBUG [RS:0;c707e9ace675:41323 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c707e9ace675:41323 2024-11-26T23:27:20,599 INFO [RS:0;c707e9ace675:41323 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:27:20,599 INFO [RS:0;c707e9ace675:41323 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:27:20,599 DEBUG [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:27:20,600 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,42489,1732663640314 with port=41323, startcode=1732663640360 2024-11-26T23:27:20,600 DEBUG [RS:0;c707e9ace675:41323 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:27:20,603 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39903, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.4 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:27:20,603 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42489 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,41323,1732663640360 2024-11-26T23:27:20,603 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42489 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,41323,1732663640360 2024-11-26T23:27:20,605 DEBUG [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c 2024-11-26T23:27:20,605 DEBUG [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34477 2024-11-26T23:27:20,605 DEBUG [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:27:20,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:27:20,607 DEBUG [RS:0;c707e9ace675:41323 {}] zookeeper.ZKUtil(111): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,41323,1732663640360 2024-11-26T23:27:20,607 WARN [RS:0;c707e9ace675:41323 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:27:20,607 INFO [RS:0;c707e9ace675:41323 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:27:20,607 DEBUG [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360 2024-11-26T23:27:20,608 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,41323,1732663640360] 2024-11-26T23:27:20,611 INFO [RS:0;c707e9ace675:41323 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:27:20,612 INFO [RS:0;c707e9ace675:41323 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:27:20,613 INFO [RS:0;c707e9ace675:41323 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:27:20,613 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,616 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:27:20,617 INFO [RS:0;c707e9ace675:41323 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:27:20,617 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,617 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,617 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,617 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,617 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,617 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,618 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:27:20,618 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,618 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,618 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,618 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,618 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,618 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:20,618 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:27:20,618 DEBUG [RS:0;c707e9ace675:41323 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:27:20,619 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,619 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,619 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,619 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,619 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,619 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,41323,1732663640360-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:27:20,635 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:27:20,635 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,41323,1732663640360-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,635 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,635 INFO [RS:0;c707e9ace675:41323 {}] regionserver.Replication(171): c707e9ace675,41323,1732663640360 started 2024-11-26T23:27:20,649 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:20,649 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,41323,1732663640360, RpcServer on c707e9ace675/172.17.0.2:41323, sessionid=0x101569aeb7b0001 2024-11-26T23:27:20,649 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:27:20,649 DEBUG [RS:0;c707e9ace675:41323 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,41323,1732663640360 2024-11-26T23:27:20,649 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,41323,1732663640360' 2024-11-26T23:27:20,649 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:27:20,650 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:27:20,650 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:27:20,650 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:27:20,650 DEBUG [RS:0;c707e9ace675:41323 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,41323,1732663640360 2024-11-26T23:27:20,650 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,41323,1732663640360' 2024-11-26T23:27:20,650 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:27:20,651 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:27:20,651 DEBUG [RS:0;c707e9ace675:41323 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:27:20,651 INFO [RS:0;c707e9ace675:41323 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:27:20,651 INFO [RS:0;c707e9ace675:41323 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:27:20,730 WARN [c707e9ace675:42489 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T23:27:20,753 INFO [RS:0;c707e9ace675:41323 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C41323%2C1732663640360, suffix=, logDir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360, archiveDir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/oldWALs, maxLogs=32 2024-11-26T23:27:20,754 INFO [RS:0;c707e9ace675:41323 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C41323%2C1732663640360.1732663640754 2024-11-26T23:27:20,759 INFO [RS:0;c707e9ace675:41323 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 2024-11-26T23:27:20,760 DEBUG [RS:0;c707e9ace675:41323 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37365:37365),(127.0.0.1/127.0.0.1:32841:32841)] 2024-11-26T23:27:20,980 DEBUG [c707e9ace675:42489 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T23:27:20,981 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c707e9ace675,41323,1732663640360 2024-11-26T23:27:20,982 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,41323,1732663640360, state=OPENING 2024-11-26T23:27:20,984 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T23:27:20,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,985 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:20,985 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:27:20,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:27:20,986 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:27:20,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,41323,1732663640360}] 2024-11-26T23:27:21,139 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:27:21,141 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47249, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:27:21,145 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T23:27:21,145 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:27:21,147 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C41323%2C1732663640360.meta, suffix=.meta, logDir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360, archiveDir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/oldWALs, maxLogs=32 2024-11-26T23:27:21,147 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C41323%2C1732663640360.meta.1732663641147.meta 2024-11-26T23:27:21,153 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.meta.1732663641147.meta 2024-11-26T23:27:21,154 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:32841:32841),(127.0.0.1/127.0.0.1:37365:37365)] 2024-11-26T23:27:21,155 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:27:21,155 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T23:27:21,155 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T23:27:21,156 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T23:27:21,156 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T23:27:21,156 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:21,156 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T23:27:21,156 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T23:27:21,157 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:27:21,158 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:27:21,158 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:21,158 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:21,159 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:27:21,159 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:27:21,159 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:21,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:21,160 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:27:21,161 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:27:21,161 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:21,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:21,161 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:27:21,162 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:27:21,162 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:21,162 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:21,162 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:27:21,163 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740 2024-11-26T23:27:21,164 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740 2024-11-26T23:27:21,165 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:21,166 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:27:21,166 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:27:21,166 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:27:21,168 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:27:21,169 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=760269, jitterRate=-0.03326864540576935}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:27:21,169 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T23:27:21,170 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732663641156Writing region info on filesystem at 1732663641156Initializing all the Stores at 1732663641157 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663641157Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663641157Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663641157Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663641157Cleaning up temporary data from old regions at 1732663641166 (+9 ms)Running coprocessor post-open hooks at 1732663641169 (+3 ms)Region opened successfully at 1732663641169 2024-11-26T23:27:21,171 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732663641139 2024-11-26T23:27:21,173 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T23:27:21,173 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T23:27:21,174 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,41323,1732663640360 2024-11-26T23:27:21,175 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,41323,1732663640360, state=OPEN 2024-11-26T23:27:21,176 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:21,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:27:21,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:27:21,182 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c707e9ace675,41323,1732663640360 2024-11-26T23:27:21,182 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:27:21,182 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:27:21,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T23:27:21,185 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,41323,1732663640360 in 196 msec 2024-11-26T23:27:21,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T23:27:21,187 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 608 msec 2024-11-26T23:27:21,188 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:27:21,188 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T23:27:21,189 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:27:21,190 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,41323,1732663640360, seqNum=-1] 2024-11-26T23:27:21,190 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:27:21,191 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45697, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:27:21,196 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 664 msec 2024-11-26T23:27:21,196 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732663641196, completionTime=-1 2024-11-26T23:27:21,196 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T23:27:21,196 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T23:27:21,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T23:27:21,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732663701198 2024-11-26T23:27:21,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732663761198 2024-11-26T23:27:21,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-26T23:27:21,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42489,1732663640314-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:21,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42489,1732663640314-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:21,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42489,1732663640314-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:21,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c707e9ace675:42489, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:21,199 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:21,199 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:21,200 DEBUG [master/c707e9ace675:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T23:27:21,202 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.803sec 2024-11-26T23:27:21,202 INFO [master/c707e9ace675:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T23:27:21,202 INFO [master/c707e9ace675:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T23:27:21,202 INFO [master/c707e9ace675:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T23:27:21,202 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T23:27:21,203 INFO [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T23:27:21,203 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42489,1732663640314-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:27:21,203 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42489,1732663640314-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T23:27:21,205 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T23:27:21,205 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T23:27:21,205 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42489,1732663640314-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:21,281 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27a17641, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:27:21,281 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,42489,-1 for getting cluster id 2024-11-26T23:27:21,281 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:27:21,283 DEBUG [HMaster-EventLoopGroup-10-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dcf238fd-6d33-450f-8e64-d44aea44b72c' 2024-11-26T23:27:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:27:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dcf238fd-6d33-450f-8e64-d44aea44b72c" 2024-11-26T23:27:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a3595db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:27:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,42489,-1] 2024-11-26T23:27:21,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:27:21,285 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:21,286 INFO [HMaster-EventLoopGroup-10-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:27:21,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd6b006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:27:21,288 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:27:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,41323,1732663640360, seqNum=-1] 2024-11-26T23:27:21,289 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:27:21,291 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-11-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34462, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:27:21,292 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c707e9ace675,42489,1732663640314 2024-11-26T23:27:21,293 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:21,295 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-26T23:27:21,295 INFO [Time-limited test {}] wal.TestLogRolling(320): Starting testLogRollOnPipelineRestart 2024-11-26T23:27:21,295 INFO [Time-limited test {}] wal.TestLogRolling(323): Replication=2 2024-11-26T23:27:21,296 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:27:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,42489,1732663640314 2024-11-26T23:27:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@160870ee 2024-11-26T23:27:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:27:21,298 INFO [HMaster-EventLoopGroup-10-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45916, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:27:21,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-26T23:27:21,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-26T23:27:21,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRollOnPipelineRestart', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:27:21,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart 2024-11-26T23:27:21,302 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:27:21,302 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:21,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRollOnPipelineRestart" procId is: 4 2024-11-26T23:27:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:27:21,303 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:27:21,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33077 is added to blk_1073741835_1011 (size=395) 2024-11-26T23:27:21,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741835_1011 (size=395) 2024-11-26T23:27:21,313 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 851db1117cbd4fd42399901d6dcad5bd, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRollOnPipelineRestart', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c 2024-11-26T23:27:21,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33325 is added to blk_1073741836_1012 (size=78) 2024-11-26T23:27:21,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33077 is added to blk_1073741836_1012 (size=78) 2024-11-26T23:27:21,321 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:21,321 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1722): Closing 851db1117cbd4fd42399901d6dcad5bd, disabling compactions & flushes 2024-11-26T23:27:21,321 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:21,321 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:21,321 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. after waiting 0 ms 2024-11-26T23:27:21,321 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:21,321 INFO [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:21,321 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRollOnPipelineRestart-pool-0 {}] regionserver.HRegion(1676): Region close journal for 851db1117cbd4fd42399901d6dcad5bd: Waiting for close lock at 1732663641321Disabling compacts and flushes for region at 1732663641321Disabling writes for close at 1732663641321Writing region close event to WAL at 1732663641321Closed at 1732663641321 2024-11-26T23:27:21,323 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:27:21,323 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd.","families":{"info":[{"qualifier":"regioninfo","vlen":77,"tag":[],"timestamp":"1732663641323"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732663641323"}]},"ts":"1732663641323"} 2024-11-26T23:27:21,326 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:27:21,327 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:27:21,327 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663641327"}]},"ts":"1732663641327"} 2024-11-26T23:27:21,329 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLING in hbase:meta 2024-11-26T23:27:21,330 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=851db1117cbd4fd42399901d6dcad5bd, ASSIGN}] 2024-11-26T23:27:21,331 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=851db1117cbd4fd42399901d6dcad5bd, ASSIGN 2024-11-26T23:27:21,332 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=851db1117cbd4fd42399901d6dcad5bd, ASSIGN; state=OFFLINE, location=c707e9ace675,41323,1732663640360; forceNewPlan=false, retain=false 2024-11-26T23:27:21,483 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=851db1117cbd4fd42399901d6dcad5bd, regionState=OPENING, regionLocation=c707e9ace675,41323,1732663640360 2024-11-26T23:27:21,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=851db1117cbd4fd42399901d6dcad5bd, ASSIGN because future has completed 2024-11-26T23:27:21,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 851db1117cbd4fd42399901d6dcad5bd, server=c707e9ace675,41323,1732663640360}] 2024-11-26T23:27:21,643 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:21,643 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 851db1117cbd4fd42399901d6dcad5bd, NAME => 'TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:27:21,644 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRollOnPipelineRestart 851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:21,644 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:21,644 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:21,644 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:21,645 INFO [StoreOpener-851db1117cbd4fd42399901d6dcad5bd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:21,647 INFO [StoreOpener-851db1117cbd4fd42399901d6dcad5bd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 851db1117cbd4fd42399901d6dcad5bd columnFamilyName info 2024-11-26T23:27:21,647 DEBUG [StoreOpener-851db1117cbd4fd42399901d6dcad5bd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:21,647 INFO [StoreOpener-851db1117cbd4fd42399901d6dcad5bd-1 {}] regionserver.HStore(327): Store=851db1117cbd4fd42399901d6dcad5bd/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:27:21,647 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:21,648 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/default/TestLogRolling-testLogRollOnPipelineRestart/851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:21,648 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/default/TestLogRolling-testLogRollOnPipelineRestart/851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:21,649 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:21,649 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:21,650 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:21,653 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/default/TestLogRolling-testLogRollOnPipelineRestart/851db1117cbd4fd42399901d6dcad5bd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:27:21,653 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 851db1117cbd4fd42399901d6dcad5bd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=827482, jitterRate=0.05219791829586029}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:27:21,653 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:21,654 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 851db1117cbd4fd42399901d6dcad5bd: Running coprocessor pre-open hook at 1732663641644Writing region info on filesystem at 1732663641644Initializing all the Stores at 1732663641645 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663641645Cleaning up temporary data from old regions at 1732663641649 (+4 ms)Running coprocessor post-open hooks at 1732663641653 (+4 ms)Region opened successfully at 1732663641654 (+1 ms) 2024-11-26T23:27:21,655 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd., pid=6, masterSystemTime=1732663641639 2024-11-26T23:27:21,658 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:21,658 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:21,659 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=851db1117cbd4fd42399901d6dcad5bd, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,41323,1732663640360 2024-11-26T23:27:21,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-11-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 851db1117cbd4fd42399901d6dcad5bd, server=c707e9ace675,41323,1732663640360 because future has completed 2024-11-26T23:27:21,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T23:27:21,665 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 851db1117cbd4fd42399901d6dcad5bd, server=c707e9ace675,41323,1732663640360 in 177 msec 2024-11-26T23:27:21,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T23:27:21,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRollOnPipelineRestart, region=851db1117cbd4fd42399901d6dcad5bd, ASSIGN in 335 msec 2024-11-26T23:27:21,669 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:27:21,669 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRollOnPipelineRestart","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663641669"}]},"ts":"1732663641669"} 2024-11-26T23:27:21,671 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRollOnPipelineRestart, state=ENABLED in hbase:meta 2024-11-26T23:27:21,672 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:27:21,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRollOnPipelineRestart in 373 msec 2024-11-26T23:27:22,166 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:22,176 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:23,166 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:23,177 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:24,167 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:24,178 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:25,167 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:25,178 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:25,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T23:27:25,734 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T23:27:25,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-26T23:27:25,735 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart Metrics about Tables on a single HBase RegionServer 2024-11-26T23:27:25,736 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:27:25,736 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T23:27:26,168 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:26,179 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:26,658 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-26T23:27:26,673 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:26,673 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:26,673 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:26,674 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:26,674 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:26,674 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:26,677 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:26,678 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:26,678 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:26,680 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:26,685 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T23:27:26,686 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRollOnPipelineRestart' 2024-11-26T23:27:27,169 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:27,179 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:28,169 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:28,180 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:29,170 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:29,180 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:30,171 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:30,181 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:31,171 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:31,181 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42489 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:27:31,357 INFO [RPCClient-NioEventLoopGroup-4-15 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRollOnPipelineRestart completed 2024-11-26T23:27:31,357 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRollOnPipelineRestart,, stopping at row=TestLogRolling-testLogRollOnPipelineRestart ,, for max=2147483647 with caching=100 2024-11-26T23:27:31,360 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRollOnPipelineRestart 2024-11-26T23:27:31,360 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:31,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRollOnPipelineRestart', row='row1002', locateType=CURRENT is [region=TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd., hostname=c707e9ace675,41323,1732663640360, seqNum=2] 2024-11-26T23:27:32,172 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:32,182 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:33,172 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:33,182 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:33,367 INFO [Time-limited test {}] wal.TestLogRolling(360): log.getCurrentFileName()): hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 2024-11-26T23:27:33,367 WARN [ResponseProcessor for block BP-1882697377-172.17.0.2-1732663639542:blk_1073741834_1010 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1882697377-172.17.0.2-1732663639542:blk_1073741834_1010 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:33,368 WARN [ResponseProcessor for block BP-1882697377-172.17.0.2-1732663639542:blk_1073741830_1006 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1882697377-172.17.0.2-1732663639542:blk_1073741830_1006 java.io.IOException: Bad response ERROR for BP-1882697377-172.17.0.2-1732663639542:blk_1073741830_1006 from datanode DatanodeInfoWithStorage[127.0.0.1:33325,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:33,368 WARN [ResponseProcessor for block BP-1882697377-172.17.0.2-1732663639542:blk_1073741833_1009 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1882697377-172.17.0.2-1732663639542:blk_1073741833_1009 java.io.IOException: Bad response ERROR for BP-1882697377-172.17.0.2-1732663639542:blk_1073741833_1009 from datanode DatanodeInfoWithStorage[127.0.0.1:33325,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1223) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:33,368 WARN [DataStreamer for file /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.meta.1732663641147.meta block BP-1882697377-172.17.0.2-1732663639542:blk_1073741834_1010 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1882697377-172.17.0.2-1732663639542:blk_1073741834_1010 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33325,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK], DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:33325,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK]) is bad. 2024-11-26T23:27:33,368 WARN [DataStreamer for file /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 block BP-1882697377-172.17.0.2-1732663639542:blk_1073741833_1009 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1882697377-172.17.0.2-1732663639542:blk_1073741833_1009 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK], DatanodeInfoWithStorage[127.0.0.1:33325,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33325,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK]) is bad. 2024-11-26T23:27:33,368 WARN [PacketResponder: BP-1882697377-172.17.0.2-1732663639542:blk_1073741833_1009, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33325] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:33,368 WARN [PacketResponder: BP-1882697377-172.17.0.2-1732663639542:blk_1073741830_1006, type=HAS_DOWNSTREAM_IN_PIPELINE, downstreams=1:[127.0.0.1:33325] {}] datanode.BlockReceiver$PacketResponder(1529): IOException in PacketResponder.run(): java.io.IOException: Connection reset by peer at sun.nio.ch.FileDispatcherImpl.write0(Native Method) ~[?:?] at sun.nio.ch.SocketDispatcher.write(SocketDispatcher.java:62) ~[?:?] at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:132) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:97) ~[?:?] at sun.nio.ch.IOUtil.write(IOUtil.java:53) ~[?:?] at sun.nio.ch.SocketChannelImpl.write(SocketChannelImpl.java:532) ~[?:?] at org.apache.hadoop.net.SocketOutputStream$Writer.performIO(SocketOutputStream.java:62) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:141) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:158) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketOutputStream.write(SocketOutputStream.java:116) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedOutputStream.flushBuffer(BufferedOutputStream.java:81) ~[?:?] at java.io.BufferedOutputStream.flush(BufferedOutputStream.java:142) ~[?:?] at java.io.DataOutputStream.flush(DataOutputStream.java:128) ~[?:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstreamUnprotected(BlockReceiver.java:1681) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.sendAckUpstream(BlockReceiver.java:1612) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1520) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:33,368 WARN [DataStreamer for file /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314/c707e9ace675%2C42489%2C1732663640314.1732663640456 block BP-1882697377-172.17.0.2-1732663639542:blk_1073741830_1006 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1882697377-172.17.0.2-1732663639542:blk_1073741830_1006 in pipeline [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK], DatanodeInfoWithStorage[127.0.0.1:33325,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK]]: datanode 1(DatanodeInfoWithStorage[127.0.0.1:33325,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK]) is bad. 2024-11-26T23:27:33,369 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_582223772_22 at /127.0.0.1:34588 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33077:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34588 dst: /127.0.0.1:33077 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:33,369 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_582223772_22 at /127.0.0.1:55602 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33325:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55602 dst: /127.0.0.1:33325 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:33,369 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_814957765_22 at /127.0.0.1:32814 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33077:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:32814 dst: /127.0.0.1:33077 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:33,369 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_582223772_22 at /127.0.0.1:55588 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33325:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55588 dst: /127.0.0.1:33325 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:33,369 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_582223772_22 at /127.0.0.1:34606 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33077:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34606 dst: /127.0.0.1:33077 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:33,369 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_814957765_22 at /127.0.0.1:47436 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33325:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47436 dst: /127.0.0.1:33325 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:33,370 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@30a928dc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:33,371 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@341f9f9e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:27:33,371 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:27:33,371 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@365c2477{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:27:33,371 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c6cb23c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,STOPPED} 2024-11-26T23:27:33,373 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:27:33,373 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:27:33,373 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1882697377-172.17.0.2-1732663639542 (Datanode Uuid 9d9cb261-b1e1-476f-85b2-4c9eadd145e1) service to localhost/127.0.0.1:34477 2024-11-26T23:27:33,373 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:27:33,373 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data3/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:33,374 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data4/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:33,374 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:27:33,384 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:33,388 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:27:33,388 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:27:33,388 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:27:33,388 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:27:33,389 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@67a2c057{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:27:33,389 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371cd9e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:27:33,509 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@269fb75c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/java.io.tmpdir/jetty-localhost-34463-hadoop-hdfs-3_4_1-tests_jar-_-any-3122788632718663528/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:33,509 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@23080db5{HTTP/1.1, (http/1.1)}{localhost:34463} 2024-11-26T23:27:33,509 INFO [Time-limited test {}] server.Server(415): Started @162854ms 2024-11-26T23:27:33,511 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:27:33,534 WARN [ResponseProcessor for block BP-1882697377-172.17.0.2-1732663639542:blk_1073741833_1014 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1882697377-172.17.0.2-1732663639542:blk_1073741833_1014 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:33,534 WARN [ResponseProcessor for block BP-1882697377-172.17.0.2-1732663639542:blk_1073741834_1015 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1882697377-172.17.0.2-1732663639542:blk_1073741834_1015 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:33,534 WARN [ResponseProcessor for block BP-1882697377-172.17.0.2-1732663639542:blk_1073741830_1013 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1882697377-172.17.0.2-1732663639542:blk_1073741830_1013 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:33,535 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_582223772_22 at /127.0.0.1:38932 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741834_1010] {}] datanode.DataXceiver(331): 127.0.0.1:33077:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38932 dst: /127.0.0.1:33077 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:33,535 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_582223772_22 at /127.0.0.1:38910 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741833_1009] {}] datanode.DataXceiver(331): 127.0.0.1:33077:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38910 dst: /127.0.0.1:33077 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:33,535 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_814957765_22 at /127.0.0.1:38926 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741830_1006] {}] datanode.DataXceiver(331): 127.0.0.1:33077:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38926 dst: /127.0.0.1:33077 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:33,540 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7096145a{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:33,541 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13bf27af{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:27:33,541 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:27:33,541 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@168478e8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:27:33,541 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@78e57521{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,STOPPED} 2024-11-26T23:27:33,542 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:27:33,543 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1882697377-172.17.0.2-1732663639542 (Datanode Uuid 0c70b437-1d71-46c0-8634-2206e7a6113e) service to localhost/127.0.0.1:34477 2024-11-26T23:27:33,543 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:27:33,543 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:27:33,544 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data1/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:33,544 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data2/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:33,544 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:27:33,583 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:33,586 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:27:33,586 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:27:33,586 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:27:33,586 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:27:33,587 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@27265229{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:27:33,587 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@156f820b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:27:33,638 WARN [Thread-1351 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:27:33,641 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9a21d940792c2a8c with lease ID 0xd0e1ecba14555497: from storage DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c node DatanodeRegistration(127.0.0.1:38833, datanodeUuid=9d9cb261-b1e1-476f-85b2-4c9eadd145e1, infoPort=40745, infoSecurePort=0, ipcPort=46217, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:33,641 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9a21d940792c2a8c with lease ID 0xd0e1ecba14555497: from storage DS-66e4ef7a-5b25-4bec-980f-55283a7b2e0b node DatanodeRegistration(127.0.0.1:38833, datanodeUuid=9d9cb261-b1e1-476f-85b2-4c9eadd145e1, infoPort=40745, infoSecurePort=0, ipcPort=46217, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:33,702 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@545fd1d9{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/java.io.tmpdir/jetty-localhost-38725-hadoop-hdfs-3_4_1-tests_jar-_-any-5092626509421219584/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:33,702 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@61726e31{HTTP/1.1, (http/1.1)}{localhost:38725} 2024-11-26T23:27:33,702 INFO [Time-limited test {}] server.Server(415): Started @163047ms 2024-11-26T23:27:33,703 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:27:33,795 WARN [Thread-1382 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:27:33,798 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x260e770aa02f4c95 with lease ID 0xd0e1ecba14555498: from storage DS-a2f3e615-428c-4f46-baa4-af09b5c855d3 node DatanodeRegistration(127.0.0.1:39663, datanodeUuid=0c70b437-1d71-46c0-8634-2206e7a6113e, infoPort=42343, infoSecurePort=0, ipcPort=46725, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:33,798 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x260e770aa02f4c95 with lease ID 0xd0e1ecba14555498: from storage DS-8fa02e10-e904-4dfe-aeae-fd152ef1dc32 node DatanodeRegistration(127.0.0.1:39663, datanodeUuid=0c70b437-1d71-46c0-8634-2206e7a6113e, infoPort=42343, infoSecurePort=0, ipcPort=46725, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:34,173 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:34,183 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:34,721 INFO [Time-limited test {}] wal.TestLogRolling(372): Data Nodes restarted 2024-11-26T23:27:34,724 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1002 2024-11-26T23:27:34,725 ERROR [FSHLog-0-hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c-prefix:c707e9ace675,41323,1732663640360 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:34,725 WARN [FSHLog-0-hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c-prefix:c707e9ace675,41323,1732663640360 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:34,725 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C41323%2C1732663640360:(num 1732663640754) roll requested 2024-11-26T23:27:34,726 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C41323%2C1732663640360.1732663654726 2024-11-26T23:27:34,731 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 newFile=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 2024-11-26T23:27:34,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:34,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:34,732 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:34,732 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:34,732 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:34,732 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 with entries=2, filesize=1.59 KB; new WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 2024-11-26T23:27:34,732 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:34,732 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:34,733 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 2024-11-26T23:27:34,733 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:42343:42343),(127.0.0.1/127.0.0.1:40745:40745)] 2024-11-26T23:27:34,733 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 is not closed yet, will try archiving it next time 2024-11-26T23:27:34,733 WARN [IPC Server handler 3 on default port 34477 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 has not been closed. Lease recovery is in progress. RecoveryId = 1017 for block blk_1073741833_1014 2024-11-26T23:27:34,733 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 after 0ms 2024-11-26T23:27:34,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39663 is added to blk_1073741833_1017 (size=1632) 2024-11-26T23:27:35,174 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:35,183 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:36,174 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:36,184 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:36,736 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1003 2024-11-26T23:27:37,175 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:37,184 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:38,175 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:38,185 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:38,641 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741833_1014: GenerationStamp not matched, existing replica is blk_1073741833_1009 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-26T23:27:38,734 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 after 4001ms 2024-11-26T23:27:38,739 WARN [ResponseProcessor for block BP-1882697377-172.17.0.2-1732663639542:blk_1073741837_1016 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1882697377-172.17.0.2-1732663639542:blk_1073741837_1016 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:38,739 WARN [DataStreamer for file /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 block BP-1882697377-172.17.0.2-1732663639542:blk_1073741837_1016 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1882697377-172.17.0.2-1732663639542:blk_1073741837_1016 in pipeline [DatanodeInfoWithStorage[127.0.0.1:39663,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK], DatanodeInfoWithStorage[127.0.0.1:38833,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:39663,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]) is bad. 2024-11-26T23:27:38,739 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_582223772_22 at /127.0.0.1:38586 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:39663:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38586 dst: /127.0.0.1:39663 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:38,740 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_582223772_22 at /127.0.0.1:40716 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:38833:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40716 dst: /127.0.0.1:38833 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:38,741 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@545fd1d9{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:38,741 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@61726e31{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:27:38,741 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:27:38,742 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@156f820b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:27:38,742 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@27265229{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,STOPPED} 2024-11-26T23:27:38,743 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:27:38,743 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:27:38,743 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:27:38,743 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1882697377-172.17.0.2-1732663639542 (Datanode Uuid 0c70b437-1d71-46c0-8634-2206e7a6113e) service to localhost/127.0.0.1:34477 2024-11-26T23:27:38,744 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data1/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:38,744 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data2/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:38,744 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:27:38,756 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:38,759 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:27:38,759 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:27:38,759 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:27:38,759 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:27:38,760 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c83d523{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:27:38,760 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a397072{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:27:38,875 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1223970{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/java.io.tmpdir/jetty-localhost-36003-hadoop-hdfs-3_4_1-tests_jar-_-any-4883508611822604743/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:38,875 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@282cfbc9{HTTP/1.1, (http/1.1)}{localhost:36003} 2024-11-26T23:27:38,875 INFO [Time-limited test {}] server.Server(415): Started @168220ms 2024-11-26T23:27:38,876 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:27:38,898 WARN [ResponseProcessor for block BP-1882697377-172.17.0.2-1732663639542:blk_1073741837_1018 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1882697377-172.17.0.2-1732663639542:blk_1073741837_1018 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:38,898 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_582223772_22 at /127.0.0.1:40724 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741837_1016] {}] datanode.DataXceiver(331): 127.0.0.1:38833:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40724 dst: /127.0.0.1:38833 java.nio.channels.ClosedChannelException: null at java.nio.channels.spi.AbstractSelectableChannel.register(AbstractSelectableChannel.java:222) ~[?:?] at java.nio.channels.SelectableChannel.register(SelectableChannel.java:260) ~[?:?] at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:334) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:38,901 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@269fb75c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:38,901 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@23080db5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:27:38,901 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:27:38,902 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371cd9e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:27:38,902 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@67a2c057{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,STOPPED} 2024-11-26T23:27:38,903 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:27:38,903 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:27:38,903 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:27:38,903 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1882697377-172.17.0.2-1732663639542 (Datanode Uuid 9d9cb261-b1e1-476f-85b2-4c9eadd145e1) service to localhost/127.0.0.1:34477 2024-11-26T23:27:38,905 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data3/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:38,905 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data4/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:38,905 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:27:38,914 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:38,917 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:27:38,919 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:27:38,919 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:27:38,919 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:27:38,920 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@42bf2aaa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:27:38,920 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1dad3af2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:27:38,979 WARN [Thread-1425 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:27:38,981 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa736e6b61ca7cb82 with lease ID 0xd0e1ecba14555499: from storage DS-a2f3e615-428c-4f46-baa4-af09b5c855d3 node DatanodeRegistration(127.0.0.1:38799, datanodeUuid=0c70b437-1d71-46c0-8634-2206e7a6113e, infoPort=37909, infoSecurePort=0, ipcPort=42793, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:38,981 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xa736e6b61ca7cb82 with lease ID 0xd0e1ecba14555499: from storage DS-8fa02e10-e904-4dfe-aeae-fd152ef1dc32 node DatanodeRegistration(127.0.0.1:38799, datanodeUuid=0c70b437-1d71-46c0-8634-2206e7a6113e, infoPort=37909, infoSecurePort=0, ipcPort=42793, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:39,038 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@474594d6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/java.io.tmpdir/jetty-localhost-38909-hadoop-hdfs-3_4_1-tests_jar-_-any-9080474065273276345/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:39,038 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3cea2bec{HTTP/1.1, (http/1.1)}{localhost:38909} 2024-11-26T23:27:39,038 INFO [Time-limited test {}] server.Server(415): Started @168383ms 2024-11-26T23:27:39,040 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:27:39,125 WARN [Thread-1456 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:27:39,127 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11033a7cdf449093 with lease ID 0xd0e1ecba1455549a: from storage DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c node DatanodeRegistration(127.0.0.1:34085, datanodeUuid=9d9cb261-b1e1-476f-85b2-4c9eadd145e1, infoPort=39871, infoSecurePort=0, ipcPort=36051, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 7, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:39,127 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x11033a7cdf449093 with lease ID 0xd0e1ecba1455549a: from storage DS-66e4ef7a-5b25-4bec-980f-55283a7b2e0b node DatanodeRegistration(127.0.0.1:34085, datanodeUuid=9d9cb261-b1e1-476f-85b2-4c9eadd145e1, infoPort=39871, infoSecurePort=0, ipcPort=36051, storageInfo=lv=-57;cid=testClusterID;nsid=386733687;c=1732663639542), blocks: 6, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:39,176 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:39,185 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:40,057 INFO [Time-limited test {}] wal.TestLogRolling(389): Data Nodes restarted 2024-11-26T23:27:40,059 INFO [Time-limited test {}] wal.AbstractTestLogRolling(330): Validated row row1004 2024-11-26T23:27:40,060 ERROR [FSHLog-0-hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c-prefix:c707e9ace675,41323,1732663640360 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38833,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:40,061 WARN [FSHLog-0-hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c-prefix:c707e9ace675,41323,1732663640360 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38833,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:40,061 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C41323%2C1732663640360:(num 1732663654726) roll requested 2024-11-26T23:27:40,061 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C41323%2C1732663640360.1732663660061 2024-11-26T23:27:40,066 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 newFile=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663660061 2024-11-26T23:27:40,066 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:40,066 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:40,066 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:40,066 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:40,066 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:40,067 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 with entries=2, filesize=2.37 KB; new WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663660061 2024-11-26T23:27:40,067 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38833,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:40,067 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:38833,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:40,067 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 2024-11-26T23:27:40,067 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37909:37909),(127.0.0.1/127.0.0.1:39871:39871)] 2024-11-26T23:27:40,067 WARN [IPC Server handler 1 on default port 34477 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 has not been closed. Lease recovery is in progress. RecoveryId = 1020 for block blk_1073741837_1018 2024-11-26T23:27:40,067 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 is not closed yet, will try archiving it next time 2024-11-26T23:27:40,068 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 after 1ms 2024-11-26T23:27:40,176 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:40,186 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:40,982 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741837_1018: GenerationStamp not matched, existing replica is blk_1073741837_1016 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-26T23:27:41,177 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:41,186 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:42,069 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C41323%2C1732663640360.1732663662069 2024-11-26T23:27:42,074 DEBUG [Time-limited test {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663660061 newFile=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 2024-11-26T23:27:42,074 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:42,074 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:42,074 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:42,075 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:42,075 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:42,075 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663660061 with entries=1, filesize=1.23 KB; new WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 2024-11-26T23:27:42,076 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39871:39871),(127.0.0.1/127.0.0.1:37909:37909)] 2024-11-26T23:27:42,076 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 is not closed yet, will try archiving it next time 2024-11-26T23:27:42,076 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663660061 is not closed yet, will try archiving it next time 2024-11-26T23:27:42,076 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 2024-11-26T23:27:42,076 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 2024-11-26T23:27:42,077 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 after 0ms 2024-11-26T23:27:42,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741838_1019 (size=1264) 2024-11-26T23:27:42,077 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 2024-11-26T23:27:42,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741838_1019 (size=1264) 2024-11-26T23:27:42,078 DEBUG [Close-WAL-Writer-1 {}] wal.AbstractFSWAL(879): hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 is not closed yet, will try archiving it next time 2024-11-26T23:27:42,086 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #3: [\x00/METAFAMILY:HBASE::REGION_EVENT::REGION_OPEN/1732663641655/Put/vlen=218/seqid=0] 2024-11-26T23:27:42,086 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #4: [row1002/info:/1732663651365/Put/vlen=1045/seqid=0] 2024-11-26T23:27:42,086 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663640754 2024-11-26T23:27:42,086 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 2024-11-26T23:27:42,086 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 2024-11-26T23:27:42,086 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 after 0ms 2024-11-26T23:27:42,086 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 2024-11-26T23:27:42,090 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #5: [row1003/info:/1732663654725/Put/vlen=1045/seqid=0] 2024-11-26T23:27:42,090 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #6: [row1004/info:/1732663656737/Put/vlen=1045/seqid=0] 2024-11-26T23:27:42,090 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 2024-11-26T23:27:42,090 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663660061 2024-11-26T23:27:42,090 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663660061 2024-11-26T23:27:42,090 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663660061 after 0ms 2024-11-26T23:27:42,090 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663660061 2024-11-26T23:27:42,093 DEBUG [Time-limited test {}] wal.TestLogRolling(412): #7: [row1005/info:/1732663660060/Put/vlen=1045/seqid=0] 2024-11-26T23:27:42,093 DEBUG [Time-limited test {}] wal.TestLogRolling(403): recovering lease for hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 2024-11-26T23:27:42,093 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 2024-11-26T23:27:42,093 WARN [IPC Server handler 4 on default port 34477 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 has not been closed. Lease recovery is in progress. RecoveryId = 1022 for block blk_1073741839_1021 2024-11-26T23:27:42,094 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 after 1ms 2024-11-26T23:27:42,177 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:42,186 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:42,986 WARN [ResponseProcessor for block BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021 {}] hdfs.DataStreamer$ResponseProcessor(1303): Exception for BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021 java.io.EOFException: Unexpected EOF while trying to read response from server at org.apache.hadoop.hdfs.protocolPB.PBHelperClient.vintPrefixed(PBHelperClient.java:529) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PipelineAck.readFields(PipelineAck.java:244) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer$ResponseProcessor.run(DataStreamer.java:1180) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:42,986 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_814957765_22 at /127.0.0.1:34272 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:34085:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34272 dst: /127.0.0.1:34085 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34085 remote=/127.0.0.1:34272]. Total timeout mills is 60000, 59088 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:42,986 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_814957765_22 at /127.0.0.1:42504 [Receiving block BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021] {}] datanode.DataXceiver(331): 127.0.0.1:38799:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42504 dst: /127.0.0.1:38799 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:27:42,986 WARN [DataStreamer for file /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 block BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021 {}] hdfs.DataStreamer(1731): Error Recovery for BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021 in pipeline [DatanodeInfoWithStorage[127.0.0.1:34085,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK], DatanodeInfoWithStorage[127.0.0.1:38799,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]]: datanode 0(DatanodeInfoWithStorage[127.0.0.1:34085,DS-284ff4e9-8601-4b12-81dc-5828c44c4d4c,DISK]) is bad. 2024-11-26T23:27:42,987 WARN [DataStreamer for file /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 block BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021 {}] hdfs.DataStreamer(859): DataStreamer Exception org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:42,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741839_1022 (size=85) 2024-11-26T23:27:43,178 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:43,187 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:44,069 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663654726 after 4002ms 2024-11-26T23:27:44,178 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:44,187 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:45,179 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:45,188 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:46,094 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 after 4001ms 2024-11-26T23:27:46,094 DEBUG [Time-limited test {}] wal.TestLogRolling(407): Reading WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 2024-11-26T23:27:46,098 DEBUG [Time-limited test {}] wal.TestLogRolling(419): EOF reading file /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 2024-11-26T23:27:46,098 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.74 KB heapSize=3.77 KB 2024-11-26T23:27:46,099 ERROR [FSHLog-0-hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c-prefix:c707e9ace675,41323,1732663640360.meta {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:46,099 WARN [FSHLog-0-hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c-prefix:c707e9ace675,41323,1732663640360.meta {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:46,099 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C41323%2C1732663640360.meta:.meta(num 1732663641147) roll requested 2024-11-26T23:27:46,099 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C41323%2C1732663640360.meta.1732663666099.meta 2024-11-26T23:27:46,104 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,104 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,104 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,104 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,104 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,105 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.meta.1732663641147.meta with entries=8, filesize=2.36 KB; new WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.meta.1732663666099.meta 2024-11-26T23:27:46,105 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:46,105 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:46,105 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.meta.1732663641147.meta 2024-11-26T23:27:46,105 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39871:39871),(127.0.0.1/127.0.0.1:37909:37909)] 2024-11-26T23:27:46,105 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(879): hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.meta.1732663641147.meta is not closed yet, will try archiving it next time 2024-11-26T23:27:46,105 WARN [IPC Server handler 3 on default port 34477 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.meta.1732663641147.meta has not been closed. Lease recovery is in progress. RecoveryId = 1024 for block blk_1073741834_1015 2024-11-26T23:27:46,106 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.meta.1732663641147.meta after 1ms 2024-11-26T23:27:46,121 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/.tmp/info/d2f3778f3de24fd09b6ba12ba45e3c38 is 207, key is TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd./info:regioninfo/1732663641659/Put/seqid=0 2024-11-26T23:27:46,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741841_1025 (size=7125) 2024-11-26T23:27:46,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741841_1025 (size=7125) 2024-11-26T23:27:46,126 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.52 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/.tmp/info/d2f3778f3de24fd09b6ba12ba45e3c38 2024-11-26T23:27:46,146 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/.tmp/ns/cbf8d796f053426ca218a138f59bd27b is 43, key is default/ns:d/1732663641191/Put/seqid=0 2024-11-26T23:27:46,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741842_1026 (size=5153) 2024-11-26T23:27:46,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741842_1026 (size=5153) 2024-11-26T23:27:46,150 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/.tmp/ns/cbf8d796f053426ca218a138f59bd27b 2024-11-26T23:27:46,168 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/.tmp/table/ebfaa50489ba4228b81ae41266dd915a is 79, key is TestLogRolling-testLogRollOnPipelineRestart/table:state/1732663641669/Put/seqid=0 2024-11-26T23:27:46,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741843_1027 (size=5438) 2024-11-26T23:27:46,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741843_1027 (size=5438) 2024-11-26T23:27:46,173 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=150 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/.tmp/table/ebfaa50489ba4228b81ae41266dd915a 2024-11-26T23:27:46,178 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/.tmp/info/d2f3778f3de24fd09b6ba12ba45e3c38 as hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/info/d2f3778f3de24fd09b6ba12ba45e3c38 2024-11-26T23:27:46,180 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:46,184 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/info/d2f3778f3de24fd09b6ba12ba45e3c38, entries=10, sequenceid=11, filesize=7.0 K 2024-11-26T23:27:46,185 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/.tmp/ns/cbf8d796f053426ca218a138f59bd27b as hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/ns/cbf8d796f053426ca218a138f59bd27b 2024-11-26T23:27:46,188 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:46,190 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/ns/cbf8d796f053426ca218a138f59bd27b, entries=2, sequenceid=11, filesize=5.0 K 2024-11-26T23:27:46,191 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/.tmp/table/ebfaa50489ba4228b81ae41266dd915a as hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/table/ebfaa50489ba4228b81ae41266dd915a 2024-11-26T23:27:46,197 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/table/ebfaa50489ba4228b81ae41266dd915a, entries=2, sequenceid=11, filesize=5.3 K 2024-11-26T23:27:46,198 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.74 KB/1782, heapSize ~3.48 KB/3560, currentSize=0 B/0 for 1588230740 in 100ms, sequenceid=11, compaction requested=false 2024-11-26T23:27:46,198 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-26T23:27:46,198 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 851db1117cbd4fd42399901d6dcad5bd 1/1 column families, dataSize=4.20 KB heapSize=4.75 KB 2024-11-26T23:27:46,199 ERROR [FSHLog-0-hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c-prefix:c707e9ace675,41323,1732663640360 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:46,199 WARN [FSHLog-0-hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c-prefix:c707e9ace675,41323,1732663640360 {}] wal.AbstractFSWAL(2174): append entry failed org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:46,199 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C41323%2C1732663640360:(num 1732663662069) roll requested 2024-11-26T23:27:46,200 INFO [regionserver/c707e9ace675:0.logRoller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C41323%2C1732663640360.1732663666200 2024-11-26T23:27:46,204 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.TestLogRolling$2(347): preLogRoll: oldFile=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 newFile=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663666200 2024-11-26T23:27:46,204 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,204 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,204 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,204 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,204 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,205 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663666200 2024-11-26T23:27:46,205 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:46,205 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. org.apache.hadoop.ipc.RemoteException: Unexpected BlockUCState: BP-1882697377-172.17.0.2-1732663639542:blk_1073741839_1021 is UNDER_RECOVERY but not UNDER_CONSTRUCTION at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkUCBlock(FSNamesystem.java:5912) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.bumpBlockGenerationStamp(FSNamesystem.java:5980) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.updateBlockForPipeline(NameNodeRpcServer.java:1002) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolServerSideTranslatorPB.java:1182) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$updateBlockForPipeline$50(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.updateBlockForPipeline(ClientNamenodeProtocolTranslatorPB.java:920) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor102.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.updateBlockForPipeline(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.DataStreamer.updateBlockForPipeline(DataStreamer.java:1786) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1657) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:46,205 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 2024-11-26T23:27:46,206 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 after 1ms 2024-11-26T23:27:46,209 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.1732663662069 to hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/oldWALs/c707e9ace675%2C41323%2C1732663640360.1732663662069 2024-11-26T23:27:46,209 DEBUG [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37909:37909),(127.0.0.1/127.0.0.1:39871:39871)] 2024-11-26T23:27:46,231 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/default/TestLogRolling-testLogRollOnPipelineRestart/851db1117cbd4fd42399901d6dcad5bd/.tmp/info/e607e45c471d4e71bfbab8fe2ecaef47 is 1080, key is row1002/info:/1732663651365/Put/seqid=0 2024-11-26T23:27:46,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741845_1029 (size=9270) 2024-11-26T23:27:46,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741845_1029 (size=9270) 2024-11-26T23:27:46,236 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.20 KB at sequenceid=8 (bloomFilter=true), to=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/default/TestLogRolling-testLogRollOnPipelineRestart/851db1117cbd4fd42399901d6dcad5bd/.tmp/info/e607e45c471d4e71bfbab8fe2ecaef47 2024-11-26T23:27:46,242 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/default/TestLogRolling-testLogRollOnPipelineRestart/851db1117cbd4fd42399901d6dcad5bd/.tmp/info/e607e45c471d4e71bfbab8fe2ecaef47 as hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/default/TestLogRolling-testLogRollOnPipelineRestart/851db1117cbd4fd42399901d6dcad5bd/info/e607e45c471d4e71bfbab8fe2ecaef47 2024-11-26T23:27:46,247 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/default/TestLogRolling-testLogRollOnPipelineRestart/851db1117cbd4fd42399901d6dcad5bd/info/e607e45c471d4e71bfbab8fe2ecaef47, entries=4, sequenceid=8, filesize=9.1 K 2024-11-26T23:27:46,248 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~4.20 KB/4304, heapSize ~4.73 KB/4848, currentSize=0 B/0 for 851db1117cbd4fd42399901d6dcad5bd in 50ms, sequenceid=8, compaction requested=false 2024-11-26T23:27:46,248 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 851db1117cbd4fd42399901d6dcad5bd: 2024-11-26T23:27:46,253 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T23:27:46,253 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:27:46,254 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:27:46,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:46,254 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:46,254 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:27:46,254 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T23:27:46,254 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1645130011, stopped=false 2024-11-26T23:27:46,254 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c707e9ace675,42489,1732663640314 2024-11-26T23:27:46,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:27:46,257 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:27:46,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:46,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:46,258 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:27:46,258 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:27:46,258 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:27:46,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:46,258 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,41323,1732663640360' ***** 2024-11-26T23:27:46,258 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:27:46,258 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T23:27:46,258 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:27:46,259 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:27:46,259 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:27:46,259 INFO [RS:0;c707e9ace675:41323 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T23:27:46,259 INFO [RS:0;c707e9ace675:41323 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T23:27:46,259 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(3091): Received CLOSE for 851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:46,259 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(959): stopping server c707e9ace675,41323,1732663640360 2024-11-26T23:27:46,260 INFO [RS:0;c707e9ace675:41323 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:27:46,260 INFO [RS:0;c707e9ace675:41323 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c707e9ace675:41323. 2024-11-26T23:27:46,260 DEBUG [RS:0;c707e9ace675:41323 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:27:46,260 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 851db1117cbd4fd42399901d6dcad5bd, disabling compactions & flushes 2024-11-26T23:27:46,260 DEBUG [RS:0;c707e9ace675:41323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:46,260 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:46,260 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:46,260 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. after waiting 0 ms 2024-11-26T23:27:46,260 INFO [RS:0;c707e9ace675:41323 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:27:46,260 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:46,260 INFO [RS:0;c707e9ace675:41323 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:27:46,260 INFO [RS:0;c707e9ace675:41323 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:27:46,260 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T23:27:46,260 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T23:27:46,260 DEBUG [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 851db1117cbd4fd42399901d6dcad5bd=TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd.} 2024-11-26T23:27:46,261 DEBUG [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 851db1117cbd4fd42399901d6dcad5bd 2024-11-26T23:27:46,261 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:27:46,261 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:27:46,261 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:27:46,261 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:27:46,261 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:27:46,265 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/default/TestLogRolling-testLogRollOnPipelineRestart/851db1117cbd4fd42399901d6dcad5bd/recovered.edits/11.seqid, newMaxSeqId=11, maxSeqId=1 2024-11-26T23:27:46,265 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:46,265 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 851db1117cbd4fd42399901d6dcad5bd: Waiting for close lock at 1732663666260Running coprocessor pre-close hooks at 1732663666260Disabling compacts and flushes for region at 1732663666260Disabling writes for close at 1732663666260Writing region close event to WAL at 1732663666261 (+1 ms)Running coprocessor post-close hooks at 1732663666265 (+4 ms)Closed at 1732663666265 2024-11-26T23:27:46,265 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRollOnPipelineRestart,,1732663641299.851db1117cbd4fd42399901d6dcad5bd. 2024-11-26T23:27:46,265 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-26T23:27:46,266 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:27:46,266 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:27:46,266 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663666260Running coprocessor pre-close hooks at 1732663666260Disabling compacts and flushes for region at 1732663666260Disabling writes for close at 1732663666261 (+1 ms)Writing region close event to WAL at 1732663666262 (+1 ms)Running coprocessor post-close hooks at 1732663666266 (+4 ms)Closed at 1732663666266 2024-11-26T23:27:46,266 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T23:27:46,461 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,41323,1732663640360; all regions closed. 2024-11-26T23:27:46,461 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,461 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,462 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,462 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,462 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:46,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741840_1023 (size=825) 2024-11-26T23:27:46,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741840_1023 (size=825) 2024-11-26T23:27:46,621 INFO [regionserver/c707e9ace675:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:27:46,621 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T23:27:46,621 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T23:27:47,180 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:47,189 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:48,181 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:48,189 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:49,181 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:49,190 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:50,106 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.meta.1732663641147.meta after 4001ms 2024-11-26T23:27:50,107 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/WALs/c707e9ace675,41323,1732663640360/c707e9ace675%2C41323%2C1732663640360.meta.1732663641147.meta to hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/oldWALs/c707e9ace675%2C41323%2C1732663640360.meta.1732663641147.meta 2024-11-26T23:27:50,110 DEBUG [RS:0;c707e9ace675:41323 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/oldWALs 2024-11-26T23:27:50,110 INFO [RS:0;c707e9ace675:41323 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C41323%2C1732663640360.meta:.meta(num 1732663666099) 2024-11-26T23:27:50,110 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,110 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,110 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,110 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,110 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741844_1028 (size=1162) 2024-11-26T23:27:50,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741844_1028 (size=1162) 2024-11-26T23:27:50,117 DEBUG [RS:0;c707e9ace675:41323 {}] wal.AbstractFSWAL(1256): Moved 4 WAL file(s) to /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/oldWALs 2024-11-26T23:27:50,117 INFO [RS:0;c707e9ace675:41323 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C41323%2C1732663640360:(num 1732663666200) 2024-11-26T23:27:50,117 DEBUG [RS:0;c707e9ace675:41323 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:50,117 INFO [RS:0;c707e9ace675:41323 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:27:50,117 INFO [RS:0;c707e9ace675:41323 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:27:50,117 INFO [RS:0;c707e9ace675:41323 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T23:27:50,118 INFO [RS:0;c707e9ace675:41323 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:27:50,118 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:27:50,118 INFO [RS:0;c707e9ace675:41323 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41323 2024-11-26T23:27:50,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,41323,1732663640360 2024-11-26T23:27:50,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:27:50,120 INFO [RS:0;c707e9ace675:41323 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:27:50,121 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,41323,1732663640360] 2024-11-26T23:27:50,122 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,41323,1732663640360 already deleted, retry=false 2024-11-26T23:27:50,123 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c707e9ace675,41323,1732663640360 expired; onlineServers=0 2024-11-26T23:27:50,123 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c707e9ace675,42489,1732663640314' ***** 2024-11-26T23:27:50,123 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T23:27:50,123 INFO [M:0;c707e9ace675:42489 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:27:50,123 INFO [M:0;c707e9ace675:42489 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:27:50,123 DEBUG [M:0;c707e9ace675:42489 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T23:27:50,123 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T23:27:50,123 DEBUG [M:0;c707e9ace675:42489 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T23:27:50,123 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663640540 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663640540,5,FailOnTimeoutGroup] 2024-11-26T23:27:50,123 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663640540 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663640540,5,FailOnTimeoutGroup] 2024-11-26T23:27:50,123 INFO [M:0;c707e9ace675:42489 {}] hbase.ChoreService(370): Chore service for: master/c707e9ace675:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T23:27:50,124 INFO [M:0;c707e9ace675:42489 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:27:50,124 DEBUG [M:0;c707e9ace675:42489 {}] master.HMaster(1795): Stopping service threads 2024-11-26T23:27:50,124 INFO [M:0;c707e9ace675:42489 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T23:27:50,124 INFO [M:0;c707e9ace675:42489 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:27:50,124 INFO [M:0;c707e9ace675:42489 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T23:27:50,124 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T23:27:50,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T23:27:50,126 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:50,126 DEBUG [M:0;c707e9ace675:42489 {}] zookeeper.ZKUtil(347): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T23:27:50,126 WARN [M:0;c707e9ace675:42489 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T23:27:50,126 INFO [M:0;c707e9ace675:42489 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/.lastflushedseqids 2024-11-26T23:27:50,128 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741834_1015: GenerationStamp not matched, existing replica is blk_1073741834_1010 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-26T23:27:50,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741846_1030 (size=139) 2024-11-26T23:27:50,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741846_1030 (size=139) 2024-11-26T23:27:50,136 INFO [M:0;c707e9ace675:42489 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T23:27:50,136 INFO [M:0;c707e9ace675:42489 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T23:27:50,137 DEBUG [M:0;c707e9ace675:42489 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:27:50,137 INFO [M:0;c707e9ace675:42489 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:50,137 DEBUG [M:0;c707e9ace675:42489 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:50,137 DEBUG [M:0;c707e9ace675:42489 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:27:50,137 DEBUG [M:0;c707e9ace675:42489 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:50,137 INFO [M:0;c707e9ace675:42489 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=23.17 KB heapSize=29.16 KB 2024-11-26T23:27:50,137 ERROR [FSHLog-0-hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData-prefix:c707e9ace675,42489,1732663640314 {}] wal.AbstractFSWAL(1838): appendAndSync throws IOException. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:50,137 WARN [FSHLog-0-hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData-prefix:c707e9ace675,42489,1732663640314 {}] wal.AbstractFSWAL(2174): append entry failed java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:50,137 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(198): WAL FSHLog c707e9ace675%2C42489%2C1732663640314:(num 1732663640456) roll requested 2024-11-26T23:27:50,138 INFO [master:store-WAL-Roller {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C42489%2C1732663640314.1732663670138 2024-11-26T23:27:50,142 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,142 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,142 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,142 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,142 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,143 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314/c707e9ace675%2C42489%2C1732663640314.1732663640456 with entries=53, filesize=26.62 KB; new WAL /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314/c707e9ace675%2C42489%2C1732663640314.1732663670138 2024-11-26T23:27:50,143 WARN [Close-WAL-Writer-0 {}] wal.AbstractProtobufLogWriter(255): Failed to write trailer, non-fatal, continuing... java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:50,143 WARN [Close-WAL-Writer-0 {}] wal.AbstractFSWAL(2043): close old writer failed. java.io.IOException: All datanodes [DatanodeInfoWithStorage[127.0.0.1:33077,DS-a2f3e615-428c-4f46-baa4-af09b5c855d3,DISK]] are bad. Aborting... at org.apache.hadoop.hdfs.DataStreamer.handleBadDatanode(DataStreamer.java:1721) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineInternal(DataStreamer.java:1644) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.setupPipelineForAppendOrRecovery(DataStreamer.java:1627) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.processDatanodeOrExternalError(DataStreamer.java:1408) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DataStreamer.run(DataStreamer.java:707) ~[hadoop-hdfs-client-3.4.1.jar:?] 2024-11-26T23:27:50,143 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314/c707e9ace675%2C42489%2C1732663640314.1732663640456 2024-11-26T23:27:50,143 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:37909:37909),(127.0.0.1/127.0.0.1:39871:39871)] 2024-11-26T23:27:50,143 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(879): hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314/c707e9ace675%2C42489%2C1732663640314.1732663640456 is not closed yet, will try archiving it next time 2024-11-26T23:27:50,144 WARN [IPC Server handler 0 on default port 34477 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314/c707e9ace675%2C42489%2C1732663640314.1732663640456 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741830_1013 2024-11-26T23:27:50,144 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314/c707e9ace675%2C42489%2C1732663640314.1732663640456 after 1ms 2024-11-26T23:27:50,158 DEBUG [M:0;c707e9ace675:42489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4576e9751f34e24b6c6516568db5889 is 82, key is hbase:meta,,1/info:regioninfo/1732663641174/Put/seqid=0 2024-11-26T23:27:50,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741848_1033 (size=5672) 2024-11-26T23:27:50,163 INFO [M:0;c707e9ace675:42489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4576e9751f34e24b6c6516568db5889 2024-11-26T23:27:50,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741848_1033 (size=5672) 2024-11-26T23:27:50,182 DEBUG [M:0;c707e9ace675:42489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6d8616319cb049b297d796b1f79bfc8a is 778, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732663641673/Put/seqid=0 2024-11-26T23:27:50,182 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:50,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741849_1034 (size=6118) 2024-11-26T23:27:50,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741849_1034 (size=6118) 2024-11-26T23:27:50,187 INFO [M:0;c707e9ace675:42489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.57 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6d8616319cb049b297d796b1f79bfc8a 2024-11-26T23:27:50,190 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:50,205 DEBUG [M:0;c707e9ace675:42489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2f036c0cec0144e488cb42a4cfe8381c is 69, key is c707e9ace675,41323,1732663640360/rs:state/1732663640603/Put/seqid=0 2024-11-26T23:27:50,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741850_1035 (size=5156) 2024-11-26T23:27:50,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741850_1035 (size=5156) 2024-11-26T23:27:50,209 INFO [M:0;c707e9ace675:42489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2f036c0cec0144e488cb42a4cfe8381c 2024-11-26T23:27:50,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:27:50,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41323-0x101569aeb7b0001, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:27:50,222 INFO [RS:0;c707e9ace675:41323 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:27:50,222 INFO [RS:0;c707e9ace675:41323 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,41323,1732663640360; zookeeper connection closed. 2024-11-26T23:27:50,222 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6ae5e010 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6ae5e010 2024-11-26T23:27:50,222 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T23:27:50,228 DEBUG [M:0;c707e9ace675:42489 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c4e96e78483241d39ed26fcdecb913c7 is 52, key is load_balancer_on/state:d/1732663641294/Put/seqid=0 2024-11-26T23:27:50,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741851_1036 (size=5056) 2024-11-26T23:27:50,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741851_1036 (size=5056) 2024-11-26T23:27:50,233 INFO [M:0;c707e9ace675:42489 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c4e96e78483241d39ed26fcdecb913c7 2024-11-26T23:27:50,238 DEBUG [M:0;c707e9ace675:42489 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/f4576e9751f34e24b6c6516568db5889 as hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f4576e9751f34e24b6c6516568db5889 2024-11-26T23:27:50,242 INFO [M:0;c707e9ace675:42489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/f4576e9751f34e24b6c6516568db5889, entries=8, sequenceid=56, filesize=5.5 K 2024-11-26T23:27:50,243 DEBUG [M:0;c707e9ace675:42489 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6d8616319cb049b297d796b1f79bfc8a as hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6d8616319cb049b297d796b1f79bfc8a 2024-11-26T23:27:50,247 INFO [M:0;c707e9ace675:42489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6d8616319cb049b297d796b1f79bfc8a, entries=6, sequenceid=56, filesize=6.0 K 2024-11-26T23:27:50,248 DEBUG [M:0;c707e9ace675:42489 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2f036c0cec0144e488cb42a4cfe8381c as hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2f036c0cec0144e488cb42a4cfe8381c 2024-11-26T23:27:50,252 INFO [M:0;c707e9ace675:42489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2f036c0cec0144e488cb42a4cfe8381c, entries=1, sequenceid=56, filesize=5.0 K 2024-11-26T23:27:50,253 DEBUG [M:0;c707e9ace675:42489 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/c4e96e78483241d39ed26fcdecb913c7 as hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c4e96e78483241d39ed26fcdecb913c7 2024-11-26T23:27:50,257 INFO [M:0;c707e9ace675:42489 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/c4e96e78483241d39ed26fcdecb913c7, entries=1, sequenceid=56, filesize=4.9 K 2024-11-26T23:27:50,258 INFO [M:0;c707e9ace675:42489 {}] regionserver.HRegion(3140): Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=56, compaction requested=false 2024-11-26T23:27:50,260 INFO [M:0;c707e9ace675:42489 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:50,260 DEBUG [M:0;c707e9ace675:42489 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663670137Disabling compacts and flushes for region at 1732663670137Disabling writes for close at 1732663670137Obtaining lock to block concurrent updates at 1732663670137Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732663670137Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=23726, getHeapSize=29800, getOffHeapSize=0, getCellsCount=67 at 1732663670137Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732663670144 (+7 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732663670144Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732663670158 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732663670158Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732663670168 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732663670181 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732663670181Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732663670191 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732663670204 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732663670204Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732663670214 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732663670228 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732663670228Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48216ece: reopening flushed file at 1732663670237 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6def0e42: reopening flushed file at 1732663670242 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cfc0dfc: reopening flushed file at 1732663670247 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31d5df40: reopening flushed file at 1732663670252 (+5 ms)Finished flush of dataSize ~23.17 KB/23726, heapSize ~29.10 KB/29800, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=56, compaction requested=false at 1732663670258 (+6 ms)Writing region close event to WAL at 1732663670260 (+2 ms)Closed at 1732663670260 2024-11-26T23:27:50,260 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,260 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,261 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,261 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,261 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:27:50,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34085 is added to blk_1073741847_1031 (size=757) 2024-11-26T23:27:50,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38799 is added to blk_1073741847_1031 (size=757) 2024-11-26T23:27:50,297 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:27:51,182 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:51,190 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:51,266 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,266 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,280 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,280 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,280 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,280 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,280 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,281 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,283 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,283 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,283 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,285 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,289 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,289 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,791 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-26T23:27:51,792 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,793 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,793 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,793 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,809 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,809 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,810 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,810 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,810 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,810 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,813 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,813 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,814 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:51,815 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:27:52,183 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:52,191 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:53,128 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1452): Error processing datanode Command java.io.IOException: Failed to delete 1 (out of 1) replica(s): 0) Failed to delete replica blk_1073741830_1013: GenerationStamp not matched, existing replica is blk_1073741830_1006 at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2389) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsDatasetImpl.invalidate(FsDatasetImpl.java:2317) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActive(BPOfferService.java:743) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPOfferService.processCommandFromActor(BPOfferService.java:692) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processCommand(BPServiceActor.java:1439) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.lambda$enqueue$2(BPServiceActor.java:1485) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1412) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) ~[hadoop-hdfs-3.4.1.jar:?] 2024-11-26T23:27:53,183 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:53,191 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:54,144 INFO [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314/c707e9ace675%2C42489%2C1732663640314.1732663640456 after 4001ms 2024-11-26T23:27:54,145 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/WALs/c707e9ace675,42489,1732663640314/c707e9ace675%2C42489%2C1732663640314.1732663640456 to hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/oldWALs/c707e9ace675%2C42489%2C1732663640314.1732663640456 2024-11-26T23:27:54,147 INFO [WAL-Archive-0 {}] region.MasterRegionUtils(50): Moved hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/MasterData/oldWALs/c707e9ace675%2C42489%2C1732663640314.1732663640456 to hdfs://localhost:34477/user/jenkins/test-data/adbf437d-39b9-f51a-9312-af15a1d7138c/oldWALs/c707e9ace675%2C42489%2C1732663640314.1732663640456$masterlocalwal$ 2024-11-26T23:27:54,147 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:27:54,147 INFO [M:0;c707e9ace675:42489 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T23:27:54,148 INFO [M:0;c707e9ace675:42489 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42489 2024-11-26T23:27:54,148 INFO [M:0;c707e9ace675:42489 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:27:54,184 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:54,192 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:54,249 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:27:54,249 INFO [M:0;c707e9ace675:42489 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:27:54,250 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42489-0x101569aeb7b0000, quorum=127.0.0.1:52559, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:27:54,252 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@474594d6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:54,253 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3cea2bec{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:27:54,253 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:27:54,253 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1dad3af2{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:27:54,253 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@42bf2aaa{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,STOPPED} 2024-11-26T23:27:54,254 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:27:54,254 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1882697377-172.17.0.2-1732663639542 (Datanode Uuid 9d9cb261-b1e1-476f-85b2-4c9eadd145e1) service to localhost/127.0.0.1:34477 2024-11-26T23:27:54,254 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:27:54,254 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:27:54,255 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data3/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:54,255 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data4/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:54,255 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:27:54,257 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1223970{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:54,257 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@282cfbc9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:27:54,257 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:27:54,257 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a397072{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:27:54,257 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c83d523{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,STOPPED} 2024-11-26T23:27:54,258 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:27:54,258 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:27:54,258 WARN [BP-1882697377-172.17.0.2-1732663639542 heartbeating to localhost/127.0.0.1:34477 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1882697377-172.17.0.2-1732663639542 (Datanode Uuid 0c70b437-1d71-46c0-8634-2206e7a6113e) service to localhost/127.0.0.1:34477 2024-11-26T23:27:54,258 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:27:54,259 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data1/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:54,259 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/cluster_ba02728d-2372-54e1-cac9-613987d4782f/data/data2/current/BP-1882697377-172.17.0.2-1732663639542 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:27:54,259 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:27:54,265 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3e84569b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:27:54,266 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@ea144a0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:27:54,266 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:27:54,266 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d60493b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:27:54,266 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6495f923{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir/,STOPPED} 2024-11-26T23:27:54,272 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T23:27:54,292 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T23:27:54,300 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnPipelineRestart Thread=179 (was 154) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34477 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34477 from jenkins.hfs.4 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34477 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:34477 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34477 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-10-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-10-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-33-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.4@localhost:34477 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:34477 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-10-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-31-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:34477 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-32-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-30-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-11-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 452) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=116 (was 180), ProcessCount=11 (was 11), AvailableMemoryMB=6717 (was 6787) 2024-11-26T23:27:54,307 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=179, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=116, ProcessCount=11, AvailableMemoryMB=6717 2024-11-26T23:27:54,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T23:27:54,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.log.dir so I do NOT create it in target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093 2024-11-26T23:27:54,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f9061e40-7573-68f2-297c-7d1ee6ea6176/hadoop.tmp.dir so I do NOT create it in target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093 2024-11-26T23:27:54,307 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d, deleteOnExit=true 2024-11-26T23:27:54,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T23:27:54,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/test.cache.data in system properties and HBase conf 2024-11-26T23:27:54,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T23:27:54,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/hadoop.log.dir in system properties and HBase conf 2024-11-26T23:27:54,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T23:27:54,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T23:27:54,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T23:27:54,308 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T23:27:54,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:27:54,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:27:54,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T23:27:54,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:27:54,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T23:27:54,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T23:27:54,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:27:54,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:27:54,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T23:27:54,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/nfs.dump.dir in system properties and HBase conf 2024-11-26T23:27:54,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/java.io.tmpdir in system properties and HBase conf 2024-11-26T23:27:54,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:27:54,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T23:27:54,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T23:27:54,322 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:27:54,391 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:54,395 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:27:54,397 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:27:54,397 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:27:54,397 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:27:54,397 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:54,398 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@56e526c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:27:54,398 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@701842fe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:27:54,511 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4682d650{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/java.io.tmpdir/jetty-localhost-37013-hadoop-hdfs-3_4_1-tests_jar-_-any-15376929920545211544/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:27:54,512 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@42501282{HTTP/1.1, (http/1.1)}{localhost:37013} 2024-11-26T23:27:54,512 INFO [Time-limited test {}] server.Server(415): Started @183857ms 2024-11-26T23:27:54,525 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:27:54,587 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:54,590 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:27:54,591 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:27:54,591 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:27:54,591 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:27:54,591 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@371e2711{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:27:54,592 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e7fac47{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:27:54,707 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@5ea88c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/java.io.tmpdir/jetty-localhost-35101-hadoop-hdfs-3_4_1-tests_jar-_-any-4648584520052826427/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:54,707 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@78617008{HTTP/1.1, (http/1.1)}{localhost:35101} 2024-11-26T23:27:54,707 INFO [Time-limited test {}] server.Server(415): Started @184052ms 2024-11-26T23:27:54,709 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:27:54,738 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:27:54,741 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:27:54,741 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:27:54,741 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:27:54,741 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:27:54,742 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3838d9cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:27:54,742 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b5fa12b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:27:54,802 WARN [Thread-1651 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d/data/data2/current/BP-2058959891-172.17.0.2-1732663674339/current, will proceed with Du for space computation calculation, 2024-11-26T23:27:54,802 WARN [Thread-1650 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d/data/data1/current/BP-2058959891-172.17.0.2-1732663674339/current, will proceed with Du for space computation calculation, 2024-11-26T23:27:54,824 WARN [Thread-1629 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:27:54,827 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd8edaab99d110f7 with lease ID 0x751dcdeba1735242: Processing first storage report for DS-086be340-4bb3-44f1-905a-eb92a7d61caa from datanode DatanodeRegistration(127.0.0.1:36941, datanodeUuid=d10d802c-e6f2-4b8f-ac38-8593ecb4d4d2, infoPort=39003, infoSecurePort=0, ipcPort=42771, storageInfo=lv=-57;cid=testClusterID;nsid=806436143;c=1732663674339) 2024-11-26T23:27:54,827 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd8edaab99d110f7 with lease ID 0x751dcdeba1735242: from storage DS-086be340-4bb3-44f1-905a-eb92a7d61caa node DatanodeRegistration(127.0.0.1:36941, datanodeUuid=d10d802c-e6f2-4b8f-ac38-8593ecb4d4d2, infoPort=39003, infoSecurePort=0, ipcPort=42771, storageInfo=lv=-57;cid=testClusterID;nsid=806436143;c=1732663674339), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:54,827 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd8edaab99d110f7 with lease ID 0x751dcdeba1735242: Processing first storage report for DS-95b4c0be-acd0-4fab-8fcd-dee2795872e8 from datanode DatanodeRegistration(127.0.0.1:36941, datanodeUuid=d10d802c-e6f2-4b8f-ac38-8593ecb4d4d2, infoPort=39003, infoSecurePort=0, ipcPort=42771, storageInfo=lv=-57;cid=testClusterID;nsid=806436143;c=1732663674339) 2024-11-26T23:27:54,827 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd8edaab99d110f7 with lease ID 0x751dcdeba1735242: from storage DS-95b4c0be-acd0-4fab-8fcd-dee2795872e8 node DatanodeRegistration(127.0.0.1:36941, datanodeUuid=d10d802c-e6f2-4b8f-ac38-8593ecb4d4d2, infoPort=39003, infoSecurePort=0, ipcPort=42771, storageInfo=lv=-57;cid=testClusterID;nsid=806436143;c=1732663674339), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:54,856 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2afda299{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/java.io.tmpdir/jetty-localhost-33645-hadoop-hdfs-3_4_1-tests_jar-_-any-2759168770063544728/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:27:54,856 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6bd1231f{HTTP/1.1, (http/1.1)}{localhost:33645} 2024-11-26T23:27:54,856 INFO [Time-limited test {}] server.Server(415): Started @184201ms 2024-11-26T23:27:54,857 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:27:54,951 WARN [Thread-1676 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d/data/data3/current/BP-2058959891-172.17.0.2-1732663674339/current, will proceed with Du for space computation calculation, 2024-11-26T23:27:54,951 WARN [Thread-1677 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d/data/data4/current/BP-2058959891-172.17.0.2-1732663674339/current, will proceed with Du for space computation calculation, 2024-11-26T23:27:54,967 WARN [Thread-1665 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:27:54,969 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7848eb9975d897db with lease ID 0x751dcdeba1735243: Processing first storage report for DS-c2d56802-34b1-4ba9-8e1c-4dfc402a1f11 from datanode DatanodeRegistration(127.0.0.1:41663, datanodeUuid=59a8e7b1-b5d3-4838-9820-62bf9620f169, infoPort=34981, infoSecurePort=0, ipcPort=39175, storageInfo=lv=-57;cid=testClusterID;nsid=806436143;c=1732663674339) 2024-11-26T23:27:54,969 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7848eb9975d897db with lease ID 0x751dcdeba1735243: from storage DS-c2d56802-34b1-4ba9-8e1c-4dfc402a1f11 node DatanodeRegistration(127.0.0.1:41663, datanodeUuid=59a8e7b1-b5d3-4838-9820-62bf9620f169, infoPort=34981, infoSecurePort=0, ipcPort=39175, storageInfo=lv=-57;cid=testClusterID;nsid=806436143;c=1732663674339), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:54,969 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7848eb9975d897db with lease ID 0x751dcdeba1735243: Processing first storage report for DS-4a47352b-c4f6-439e-a196-bec4c3b34344 from datanode DatanodeRegistration(127.0.0.1:41663, datanodeUuid=59a8e7b1-b5d3-4838-9820-62bf9620f169, infoPort=34981, infoSecurePort=0, ipcPort=39175, storageInfo=lv=-57;cid=testClusterID;nsid=806436143;c=1732663674339) 2024-11-26T23:27:54,969 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7848eb9975d897db with lease ID 0x751dcdeba1735243: from storage DS-4a47352b-c4f6-439e-a196-bec4c3b34344 node DatanodeRegistration(127.0.0.1:41663, datanodeUuid=59a8e7b1-b5d3-4838-9820-62bf9620f169, infoPort=34981, infoSecurePort=0, ipcPort=39175, storageInfo=lv=-57;cid=testClusterID;nsid=806436143;c=1732663674339), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:27:54,979 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093 2024-11-26T23:27:54,981 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d/zookeeper_0, clientPort=58917, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T23:27:54,982 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58917 2024-11-26T23:27:54,982 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:54,983 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:54,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:27:54,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:27:54,992 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de with version=8 2024-11-26T23:27:54,993 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/hbase-staging 2024-11-26T23:27:54,994 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:27:54,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:54,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:54,995 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:27:54,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:54,995 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:27:54,995 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T23:27:54,995 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:27:54,996 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34975 2024-11-26T23:27:54,998 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34975 connecting to ZooKeeper ensemble=127.0.0.1:58917 2024-11-26T23:27:55,004 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:349750x0, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:27:55,004 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34975-0x101569b72f40000 connected 2024-11-26T23:27:55,022 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:55,023 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:55,025 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:27:55,025 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de, hbase.cluster.distributed=false 2024-11-26T23:27:55,027 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:27:55,027 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34975 2024-11-26T23:27:55,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34975 2024-11-26T23:27:55,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34975 2024-11-26T23:27:55,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34975 2024-11-26T23:27:55,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34975 2024-11-26T23:27:55,044 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:27:55,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:55,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:55,044 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:27:55,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:27:55,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:27:55,044 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:27:55,044 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:27:55,045 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45123 2024-11-26T23:27:55,046 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45123 connecting to ZooKeeper ensemble=127.0.0.1:58917 2024-11-26T23:27:55,047 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:55,049 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:55,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:451230x0, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:27:55,053 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:451230x0, quorum=127.0.0.1:58917, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:27:55,053 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45123-0x101569b72f40001 connected 2024-11-26T23:27:55,053 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:27:55,054 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:27:55,054 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T23:27:55,055 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:27:55,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45123 2024-11-26T23:27:55,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45123 2024-11-26T23:27:55,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45123 2024-11-26T23:27:55,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45123 2024-11-26T23:27:55,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45123 2024-11-26T23:27:55,068 DEBUG [M:0;c707e9ace675:34975 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c707e9ace675:34975 2024-11-26T23:27:55,069 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c707e9ace675,34975,1732663674994 2024-11-26T23:27:55,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:27:55,070 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:27:55,071 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c707e9ace675,34975,1732663674994 2024-11-26T23:27:55,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T23:27:55,072 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,073 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T23:27:55,073 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c707e9ace675,34975,1732663674994 from backup master directory 2024-11-26T23:27:55,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c707e9ace675,34975,1732663674994 2024-11-26T23:27:55,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:27:55,076 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:27:55,076 WARN [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:27:55,076 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c707e9ace675,34975,1732663674994 2024-11-26T23:27:55,079 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/hbase.id] with ID: f5c4a587-6b0f-49e5-ae8c-49f323e4b69e 2024-11-26T23:27:55,079 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/.tmp/hbase.id 2024-11-26T23:27:55,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:27:55,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:27:55,085 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/.tmp/hbase.id]:[hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/hbase.id] 2024-11-26T23:27:55,096 INFO [master/c707e9ace675:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:55,096 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T23:27:55,098 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 2ms. 2024-11-26T23:27:55,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:27:55,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:27:55,108 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:27:55,109 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T23:27:55,109 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:27:55,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:27:55,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:27:55,116 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store 2024-11-26T23:27:55,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:27:55,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:27:55,126 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:55,126 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:27:55,126 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:55,126 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:55,126 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:27:55,126 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:55,126 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:27:55,126 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663675126Disabling compacts and flushes for region at 1732663675126Disabling writes for close at 1732663675126Writing region close event to WAL at 1732663675126Closed at 1732663675126 2024-11-26T23:27:55,127 WARN [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/.initializing 2024-11-26T23:27:55,127 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/WALs/c707e9ace675,34975,1732663674994 2024-11-26T23:27:55,130 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C34975%2C1732663674994, suffix=, logDir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/WALs/c707e9ace675,34975,1732663674994, archiveDir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/oldWALs, maxLogs=10 2024-11-26T23:27:55,130 INFO [master/c707e9ace675:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C34975%2C1732663674994.1732663675130 2024-11-26T23:27:55,135 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/WALs/c707e9ace675,34975,1732663674994/c707e9ace675%2C34975%2C1732663674994.1732663675130 2024-11-26T23:27:55,135 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34981:34981),(127.0.0.1/127.0.0.1:39003:39003)] 2024-11-26T23:27:55,136 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:27:55,136 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:55,136 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,136 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,137 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T23:27:55,139 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:55,139 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T23:27:55,140 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:27:55,140 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,141 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T23:27:55,141 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:27:55,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,142 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T23:27:55,143 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,143 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:27:55,143 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,144 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,144 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,145 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,145 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,146 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:27:55,147 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:27:55,149 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:27:55,149 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=703493, jitterRate=-0.1054629236459732}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:27:55,150 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732663675136Initializing all the Stores at 1732663675137 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663675137Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663675137Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663675137Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663675137Cleaning up temporary data from old regions at 1732663675145 (+8 ms)Region opened successfully at 1732663675150 (+5 ms) 2024-11-26T23:27:55,150 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T23:27:55,153 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9aaf81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:27:55,154 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T23:27:55,154 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T23:27:55,154 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T23:27:55,154 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T23:27:55,154 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T23:27:55,154 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-26T23:27:55,155 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T23:27:55,156 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T23:27:55,157 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T23:27:55,158 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T23:27:55,159 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T23:27:55,159 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T23:27:55,160 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T23:27:55,161 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T23:27:55,162 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T23:27:55,164 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T23:27:55,165 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T23:27:55,166 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T23:27:55,168 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T23:27:55,169 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T23:27:55,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:27:55,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:27:55,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,171 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c707e9ace675,34975,1732663674994, sessionid=0x101569b72f40000, setting cluster-up flag (Was=false) 2024-11-26T23:27:55,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,174 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,179 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T23:27:55,180 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,34975,1732663674994 2024-11-26T23:27:55,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,183 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,184 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:55,189 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T23:27:55,190 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,34975,1732663674994 2024-11-26T23:27:55,191 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T23:27:55,192 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:55,193 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T23:27:55,193 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T23:27:55,193 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T23:27:55,193 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c707e9ace675,34975,1732663674994 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T23:27:55,194 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:27:55,195 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:27:55,195 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:27:55,195 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:27:55,195 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c707e9ace675:0, corePoolSize=10, maxPoolSize=10 2024-11-26T23:27:55,195 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,195 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:27:55,195 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,197 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732663705197 2024-11-26T23:27:55,197 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T23:27:55,197 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:27:55,197 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T23:27:55,197 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T23:27:55,197 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T23:27:55,197 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T23:27:55,197 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T23:27:55,197 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T23:27:55,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T23:27:55,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T23:27:55,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T23:27:55,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T23:27:55,198 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,198 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T23:27:55,198 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T23:27:55,199 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663675198,5,FailOnTimeoutGroup] 2024-11-26T23:27:55,199 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663675199,5,FailOnTimeoutGroup] 2024-11-26T23:27:55,199 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,200 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T23:27:55,200 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,200 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:27:55,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:27:55,207 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T23:27:55,207 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de 2024-11-26T23:27:55,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:27:55,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:27:55,217 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:55,218 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:27:55,219 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:27:55,219 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,220 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:55,220 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:27:55,221 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:27:55,221 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,221 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:55,222 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:27:55,222 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:27:55,223 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:55,223 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:27:55,224 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:27:55,224 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,224 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:55,224 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:27:55,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740 2024-11-26T23:27:55,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740 2024-11-26T23:27:55,226 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:27:55,226 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:27:55,227 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:27:55,228 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:27:55,229 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:27:55,230 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=850697, jitterRate=0.08171840012073517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:27:55,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732663675217Initializing all the Stores at 1732663675218 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663675218Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663675218Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663675218Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663675218Cleaning up temporary data from old regions at 1732663675226 (+8 ms)Region opened successfully at 1732663675230 (+4 ms) 2024-11-26T23:27:55,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:27:55,230 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:27:55,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:27:55,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:27:55,230 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:27:55,231 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:27:55,231 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663675230Disabling compacts and flushes for region at 1732663675230Disabling writes for close at 1732663675230Writing region close event to WAL at 1732663675231 (+1 ms)Closed at 1732663675231 2024-11-26T23:27:55,232 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:27:55,232 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T23:27:55,232 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T23:27:55,234 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:27:55,235 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T23:27:55,258 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(746): ClusterId : f5c4a587-6b0f-49e5-ae8c-49f323e4b69e 2024-11-26T23:27:55,258 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:27:55,260 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:27:55,260 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:27:55,262 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:27:55,263 DEBUG [RS:0;c707e9ace675:45123 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3a9a06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:27:55,274 DEBUG [RS:0;c707e9ace675:45123 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c707e9ace675:45123 2024-11-26T23:27:55,274 INFO [RS:0;c707e9ace675:45123 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:27:55,274 INFO [RS:0;c707e9ace675:45123 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:27:55,274 DEBUG [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:27:55,275 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,34975,1732663674994 with port=45123, startcode=1732663675044 2024-11-26T23:27:55,275 DEBUG [RS:0;c707e9ace675:45123 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:27:55,277 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47247, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.5 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:27:55,278 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34975 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,45123,1732663675044 2024-11-26T23:27:55,278 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34975 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,45123,1732663675044 2024-11-26T23:27:55,279 DEBUG [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de 2024-11-26T23:27:55,280 DEBUG [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39031 2024-11-26T23:27:55,280 DEBUG [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:27:55,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:27:55,282 DEBUG [RS:0;c707e9ace675:45123 {}] zookeeper.ZKUtil(111): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,45123,1732663675044 2024-11-26T23:27:55,282 WARN [RS:0;c707e9ace675:45123 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:27:55,282 INFO [RS:0;c707e9ace675:45123 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:27:55,283 DEBUG [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044 2024-11-26T23:27:55,283 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,45123,1732663675044] 2024-11-26T23:27:55,286 INFO [RS:0;c707e9ace675:45123 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:27:55,287 INFO [RS:0;c707e9ace675:45123 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:27:55,288 INFO [RS:0;c707e9ace675:45123 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:27:55,288 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,288 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:27:55,288 INFO [RS:0;c707e9ace675:45123 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:27:55,289 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:27:55,289 DEBUG [RS:0;c707e9ace675:45123 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:27:55,290 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,290 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,290 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,290 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,290 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,291 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,45123,1732663675044-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:27:55,313 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:27:55,313 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,45123,1732663675044-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,313 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,313 INFO [RS:0;c707e9ace675:45123 {}] regionserver.Replication(171): c707e9ace675,45123,1732663675044 started 2024-11-26T23:27:55,331 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,332 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,45123,1732663675044, RpcServer on c707e9ace675/172.17.0.2:45123, sessionid=0x101569b72f40001 2024-11-26T23:27:55,332 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:27:55,332 DEBUG [RS:0;c707e9ace675:45123 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,45123,1732663675044 2024-11-26T23:27:55,332 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,45123,1732663675044' 2024-11-26T23:27:55,332 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:27:55,332 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:27:55,333 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:27:55,333 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:27:55,333 DEBUG [RS:0;c707e9ace675:45123 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,45123,1732663675044 2024-11-26T23:27:55,333 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,45123,1732663675044' 2024-11-26T23:27:55,333 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:27:55,333 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:27:55,334 DEBUG [RS:0;c707e9ace675:45123 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:27:55,334 INFO [RS:0;c707e9ace675:45123 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:27:55,334 INFO [RS:0;c707e9ace675:45123 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:27:55,385 WARN [c707e9ace675:34975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T23:27:55,436 INFO [RS:0;c707e9ace675:45123 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C45123%2C1732663675044, suffix=, logDir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044, archiveDir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/oldWALs, maxLogs=32 2024-11-26T23:27:55,436 INFO [RS:0;c707e9ace675:45123 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C45123%2C1732663675044.1732663675436 2024-11-26T23:27:55,442 INFO [RS:0;c707e9ace675:45123 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663675436 2024-11-26T23:27:55,443 DEBUG [RS:0;c707e9ace675:45123 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34981:34981),(127.0.0.1/127.0.0.1:39003:39003)] 2024-11-26T23:27:55,635 DEBUG [c707e9ace675:34975 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T23:27:55,636 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c707e9ace675,45123,1732663675044 2024-11-26T23:27:55,637 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,45123,1732663675044, state=OPENING 2024-11-26T23:27:55,639 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T23:27:55,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:27:55,640 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:27:55,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,45123,1732663675044}] 2024-11-26T23:27:55,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:27:55,641 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:27:55,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:27:55,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T23:27:55,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRollOnPipelineRestart 2024-11-26T23:27:55,794 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:27:55,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59861, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:27:55,799 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T23:27:55,799 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:27:55,801 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C45123%2C1732663675044.meta, suffix=.meta, logDir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044, archiveDir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/oldWALs, maxLogs=32 2024-11-26T23:27:55,802 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C45123%2C1732663675044.meta.1732663675801.meta 2024-11-26T23:27:55,813 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.meta.1732663675801.meta 2024-11-26T23:27:55,816 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34981:34981),(127.0.0.1/127.0.0.1:39003:39003)] 2024-11-26T23:27:55,817 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:27:55,817 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T23:27:55,818 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T23:27:55,818 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T23:27:55,818 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T23:27:55,818 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:55,818 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T23:27:55,818 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T23:27:55,819 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:27:55,820 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:27:55,820 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:55,821 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:27:55,821 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:27:55,821 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,822 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:55,822 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:27:55,822 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:27:55,822 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:55,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:27:55,823 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:27:55,823 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,824 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:27:55,824 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:27:55,824 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740 2024-11-26T23:27:55,825 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740 2024-11-26T23:27:55,827 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:27:55,827 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:27:55,827 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:27:55,828 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:27:55,829 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=869818, jitterRate=0.10603086650371552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:27:55,829 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T23:27:55,830 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732663675818Writing region info on filesystem at 1732663675818Initializing all the Stores at 1732663675819 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663675819Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663675819Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663675819Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663675819Cleaning up temporary data from old regions at 1732663675827 (+8 ms)Running coprocessor post-open hooks at 1732663675829 (+2 ms)Region opened successfully at 1732663675830 (+1 ms) 2024-11-26T23:27:55,831 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732663675794 2024-11-26T23:27:55,833 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T23:27:55,833 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T23:27:55,834 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,45123,1732663675044 2024-11-26T23:27:55,836 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,45123,1732663675044, state=OPEN 2024-11-26T23:27:55,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:27:55,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:27:55,844 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c707e9ace675,45123,1732663675044 2024-11-26T23:27:55,844 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:27:55,844 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:27:55,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T23:27:55,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,45123,1732663675044 in 204 msec 2024-11-26T23:27:55,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T23:27:55,849 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 615 msec 2024-11-26T23:27:55,850 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:27:55,850 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T23:27:55,852 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:27:55,852 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,45123,1732663675044, seqNum=-1] 2024-11-26T23:27:55,852 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:27:55,853 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59755, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:27:55,859 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 666 msec 2024-11-26T23:27:55,859 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732663675859, completionTime=-1 2024-11-26T23:27:55,859 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T23:27:55,860 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T23:27:55,861 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T23:27:55,861 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732663735861 2024-11-26T23:27:55,861 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732663795861 2024-11-26T23:27:55,861 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-26T23:27:55,862 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34975,1732663674994-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,862 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34975,1732663674994-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,862 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34975,1732663674994-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,862 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c707e9ace675:34975, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,862 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,862 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,864 DEBUG [master/c707e9ace675:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T23:27:55,865 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.789sec 2024-11-26T23:27:55,865 INFO [master/c707e9ace675:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T23:27:55,865 INFO [master/c707e9ace675:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T23:27:55,865 INFO [master/c707e9ace675:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T23:27:55,865 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T23:27:55,865 INFO [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T23:27:55,865 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34975,1732663674994-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:27:55,865 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34975,1732663674994-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T23:27:55,867 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T23:27:55,867 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T23:27:55,868 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,34975,1732663674994-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:27:55,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0f2114, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:27:55,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,34975,-1 for getting cluster id 2024-11-26T23:27:55,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:27:55,961 DEBUG [HMaster-EventLoopGroup-12-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f5c4a587-6b0f-49e5-ae8c-49f323e4b69e' 2024-11-26T23:27:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:27:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f5c4a587-6b0f-49e5-ae8c-49f323e4b69e" 2024-11-26T23:27:55,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3892c0bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:27:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,34975,-1] 2024-11-26T23:27:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:27:55,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:27:55,963 INFO [HMaster-EventLoopGroup-12-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:27:55,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd49fe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:27:55,964 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:27:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,45123,1732663675044, seqNum=-1] 2024-11-26T23:27:55,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:27:55,967 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-13-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:27:55,969 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c707e9ace675,34975,1732663674994 2024-11-26T23:27:55,969 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:27:55,972 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-26T23:27:55,973 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:27:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,34975,1732663674994 2024-11-26T23:27:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c332338 2024-11-26T23:27:55,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:27:55,975 INFO [HMaster-EventLoopGroup-12-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52796, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:27:55,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-26T23:27:55,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-26T23:27:55,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testCompactionRecordDoesntBlockRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:27:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:27:55,978 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:27:55,978 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:55,978 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testCompactionRecordDoesntBlockRolling" procId is: 4 2024-11-26T23:27:55,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:27:55,979 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:27:55,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741835_1011 (size=405) 2024-11-26T23:27:55,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741835_1011 (size=405) 2024-11-26T23:27:55,987 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e6b9d66717927e4826acc23a0a81da43, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testCompactionRecordDoesntBlockRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de 2024-11-26T23:27:55,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741836_1012 (size=88) 2024-11-26T23:27:55,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741836_1012 (size=88) 2024-11-26T23:27:55,994 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:55,994 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1722): Closing e6b9d66717927e4826acc23a0a81da43, disabling compactions & flushes 2024-11-26T23:27:55,994 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:27:55,994 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:27:55,994 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. after waiting 0 ms 2024-11-26T23:27:55,994 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:27:55,994 INFO [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:27:55,994 DEBUG [RegionOpenAndInit-TestLogRolling-testCompactionRecordDoesntBlockRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for e6b9d66717927e4826acc23a0a81da43: Waiting for close lock at 1732663675994Disabling compacts and flushes for region at 1732663675994Disabling writes for close at 1732663675994Writing region close event to WAL at 1732663675994Closed at 1732663675994 2024-11-26T23:27:55,995 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:27:55,996 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.","families":{"info":[{"qualifier":"regioninfo","vlen":87,"tag":[],"timestamp":"1732663675995"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732663675995"}]},"ts":"1732663675995"} 2024-11-26T23:27:55,998 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:27:55,999 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:27:56,000 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663676000"}]},"ts":"1732663676000"} 2024-11-26T23:27:56,002 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLING in hbase:meta 2024-11-26T23:27:56,002 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e6b9d66717927e4826acc23a0a81da43, ASSIGN}] 2024-11-26T23:27:56,004 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e6b9d66717927e4826acc23a0a81da43, ASSIGN 2024-11-26T23:27:56,005 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e6b9d66717927e4826acc23a0a81da43, ASSIGN; state=OFFLINE, location=c707e9ace675,45123,1732663675044; forceNewPlan=false, retain=false 2024-11-26T23:27:56,155 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e6b9d66717927e4826acc23a0a81da43, regionState=OPENING, regionLocation=c707e9ace675,45123,1732663675044 2024-11-26T23:27:56,158 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e6b9d66717927e4826acc23a0a81da43, ASSIGN because future has completed 2024-11-26T23:27:56,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6b9d66717927e4826acc23a0a81da43, server=c707e9ace675,45123,1732663675044}] 2024-11-26T23:27:56,185 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:56,193 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:56,314 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:27:56,315 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => e6b9d66717927e4826acc23a0a81da43, NAME => 'TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:27:56,315 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testCompactionRecordDoesntBlockRolling e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:27:56,315 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:27:56,315 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:27:56,315 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:27:56,316 INFO [StoreOpener-e6b9d66717927e4826acc23a0a81da43-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:27:56,318 INFO [StoreOpener-e6b9d66717927e4826acc23a0a81da43-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e6b9d66717927e4826acc23a0a81da43 columnFamilyName info 2024-11-26T23:27:56,318 DEBUG [StoreOpener-e6b9d66717927e4826acc23a0a81da43-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:27:56,318 INFO [StoreOpener-e6b9d66717927e4826acc23a0a81da43-1 {}] regionserver.HStore(327): Store=e6b9d66717927e4826acc23a0a81da43/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:27:56,318 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:27:56,319 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:27:56,319 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:27:56,320 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:27:56,320 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:27:56,321 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:27:56,323 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:27:56,323 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened e6b9d66717927e4826acc23a0a81da43; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=850625, jitterRate=0.08162565529346466}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:27:56,323 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:27:56,324 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for e6b9d66717927e4826acc23a0a81da43: Running coprocessor pre-open hook at 1732663676315Writing region info on filesystem at 1732663676315Initializing all the Stores at 1732663676316 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663676316Cleaning up temporary data from old regions at 1732663676320 (+4 ms)Running coprocessor post-open hooks at 1732663676323 (+3 ms)Region opened successfully at 1732663676324 (+1 ms) 2024-11-26T23:27:56,325 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43., pid=6, masterSystemTime=1732663676311 2024-11-26T23:27:56,327 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:27:56,327 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:27:56,328 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=e6b9d66717927e4826acc23a0a81da43, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,45123,1732663675044 2024-11-26T23:27:56,330 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-13-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure e6b9d66717927e4826acc23a0a81da43, server=c707e9ace675,45123,1732663675044 because future has completed 2024-11-26T23:27:56,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T23:27:56,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure e6b9d66717927e4826acc23a0a81da43, server=c707e9ace675,45123,1732663675044 in 174 msec 2024-11-26T23:27:56,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T23:27:56,336 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling, region=e6b9d66717927e4826acc23a0a81da43, ASSIGN in 332 msec 2024-11-26T23:27:56,337 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:27:56,337 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testCompactionRecordDoesntBlockRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663676337"}]},"ts":"1732663676337"} 2024-11-26T23:27:56,339 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testCompactionRecordDoesntBlockRolling, state=ENABLED in hbase:meta 2024-11-26T23:27:56,340 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:27:56,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 365 msec 2024-11-26T23:27:57,185 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:57,193 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:58,186 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:58,193 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:59,186 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:27:59,194 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:00,187 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:00,194 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:01,188 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:01,195 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:01,236 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-26T23:28:01,237 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,237 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,238 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,238 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,238 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,238 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,254 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,255 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,255 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,255 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,255 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,256 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,259 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,259 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,259 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,261 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:01,286 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T23:28:01,286 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testCompactionRecordDoesntBlockRolling' 2024-11-26T23:28:02,188 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:02,195 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:03,189 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:03,195 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:04,189 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:04,196 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:05,190 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:05,196 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:05,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T23:28:05,734 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-26T23:28:05,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:28:05,735 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-26T23:28:05,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:28:05,735 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling Metrics about Tables on a single HBase RegionServer 2024-11-26T23:28:06,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:28:06,007 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-26T23:28:06,007 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testCompactionRecordDoesntBlockRolling,, stopping at row=TestLogRolling-testCompactionRecordDoesntBlockRolling ,, for max=2147483647 with caching=100 2024-11-26T23:28:06,010 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:28:06,010 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:06,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testCompactionRecordDoesntBlockRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43., hostname=c707e9ace675,45123,1732663675044, seqNum=2] 2024-11-26T23:28:06,020 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:28:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:28:06,025 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:28:06,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T23:28:06,026 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:28:06,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:28:06,187 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-26T23:28:06,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:06,188 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing e6b9d66717927e4826acc23a0a81da43 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-26T23:28:06,190 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:06,196 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:06,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/27b1de05063140cf91e93de6c40568c2 is 1080, key is row0001/info:/1732663686014/Put/seqid=0 2024-11-26T23:28:06,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741837_1013 (size=6033) 2024-11-26T23:28:06,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741837_1013 (size=6033) 2024-11-26T23:28:06,210 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/27b1de05063140cf91e93de6c40568c2 2024-11-26T23:28:06,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/27b1de05063140cf91e93de6c40568c2 as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/27b1de05063140cf91e93de6c40568c2 2024-11-26T23:28:06,222 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/27b1de05063140cf91e93de6c40568c2, entries=1, sequenceid=5, filesize=5.9 K 2024-11-26T23:28:06,223 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e6b9d66717927e4826acc23a0a81da43 in 35ms, sequenceid=5, compaction requested=false 2024-11-26T23:28:06,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for e6b9d66717927e4826acc23a0a81da43: 2024-11-26T23:28:06,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:06,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-26T23:28:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-26T23:28:06,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T23:28:06,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 200 msec 2024-11-26T23:28:06,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 210 msec 2024-11-26T23:28:07,191 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:07,197 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:08,191 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:08,197 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:09,192 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:09,198 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:10,193 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:10,198 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:11,193 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:11,198 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:12,194 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:12,199 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:13,194 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:13,199 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:14,195 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:14,199 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:15,196 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:15,200 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-26T23:28:16,078 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-26T23:28:16,081 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:28:16,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:28:16,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T23:28:16,083 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:28:16,084 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:28:16,084 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:28:16,196 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:16,200 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:16,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-26T23:28:16,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:16,238 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing e6b9d66717927e4826acc23a0a81da43 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-26T23:28:16,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/e41dbc32be3a4a54a4cb774d3abbcf04 is 1080, key is row0002/info:/1732663696079/Put/seqid=0 2024-11-26T23:28:16,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741838_1014 (size=6033) 2024-11-26T23:28:16,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741838_1014 (size=6033) 2024-11-26T23:28:16,249 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/e41dbc32be3a4a54a4cb774d3abbcf04 2024-11-26T23:28:16,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/e41dbc32be3a4a54a4cb774d3abbcf04 as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e41dbc32be3a4a54a4cb774d3abbcf04 2024-11-26T23:28:16,260 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e41dbc32be3a4a54a4cb774d3abbcf04, entries=1, sequenceid=9, filesize=5.9 K 2024-11-26T23:28:16,261 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e6b9d66717927e4826acc23a0a81da43 in 23ms, sequenceid=9, compaction requested=false 2024-11-26T23:28:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for e6b9d66717927e4826acc23a0a81da43: 2024-11-26T23:28:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:16,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-26T23:28:16,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-26T23:28:16,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-26T23:28:16,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 179 msec 2024-11-26T23:28:16,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 185 msec 2024-11-26T23:28:17,197 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:17,201 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:18,197 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:18,201 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:19,198 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:19,201 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:20,198 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:20,202 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:21,199 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:21,199 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 after 68044ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor193.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:28:21,202 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:21,202 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(233): attempt=2 on file=hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta after 68029ms java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.recoverLease(DFSClient.java:946) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:317) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$3.doCall(DistributedFileSystem.java:314) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.recoverLease(DistributedFileSystem.java:329) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor193.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverLease(RecoverLeaseFSUtils.java:222) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:155) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-26T23:28:22,200 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:22,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:23,200 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:23,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:24,201 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:24,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:24,978 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:28:25,201 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:25,204 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-26T23:28:26,097 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-26T23:28:26,100 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C45123%2C1732663675044.1732663706100 2024-11-26T23:28:26,106 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:26,106 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:26,106 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:26,106 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:26,106 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:26,106 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663675436 with entries=8, filesize=5.41 KB; new WAL /user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663706100 2024-11-26T23:28:26,107 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39003:39003),(127.0.0.1/127.0.0.1:34981:34981)] 2024-11-26T23:28:26,107 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663675436 is not closed yet, will try archiving it next time 2024-11-26T23:28:26,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741833_1009 (size=5546) 2024-11-26T23:28:26,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741833_1009 (size=5546) 2024-11-26T23:28:26,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:28:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:28:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T23:28:26,112 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:28:26,113 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:28:26,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:28:26,202 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:26,204 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:26,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-26T23:28:26,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:26,267 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing e6b9d66717927e4826acc23a0a81da43 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-26T23:28:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/e2b56dc34e5c4d9b9b417a77282b7051 is 1080, key is row0003/info:/1732663706098/Put/seqid=0 2024-11-26T23:28:26,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741840_1016 (size=6033) 2024-11-26T23:28:26,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741840_1016 (size=6033) 2024-11-26T23:28:26,277 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/e2b56dc34e5c4d9b9b417a77282b7051 2024-11-26T23:28:26,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/e2b56dc34e5c4d9b9b417a77282b7051 as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e2b56dc34e5c4d9b9b417a77282b7051 2024-11-26T23:28:26,288 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e2b56dc34e5c4d9b9b417a77282b7051, entries=1, sequenceid=13, filesize=5.9 K 2024-11-26T23:28:26,290 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e6b9d66717927e4826acc23a0a81da43 in 23ms, sequenceid=13, compaction requested=true 2024-11-26T23:28:26,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for e6b9d66717927e4826acc23a0a81da43: 2024-11-26T23:28:26,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:26,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-26T23:28:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-26T23:28:26,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-26T23:28:26,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 179 msec 2024-11-26T23:28:26,296 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 186 msec 2024-11-26T23:28:27,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:27,204 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:28,203 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:28,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:29,204 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:29,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:30,204 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:30,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:31,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:31,206 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:32,205 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:32,206 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:33,206 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:33,206 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:34,206 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:34,207 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:35,207 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:35,207 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-26T23:28:36,187 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-26T23:28:36,187 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:28:36,188 DEBUG [Time-limited test {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 18099 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:28:36,188 DEBUG [Time-limited test {}] regionserver.HStore(1541): e6b9d66717927e4826acc23a0a81da43/info is initiating minor compaction (all files) 2024-11-26T23:28:36,189 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:28:36,189 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:36,189 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e6b9d66717927e4826acc23a0a81da43/info in TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:36,189 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/27b1de05063140cf91e93de6c40568c2, hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e41dbc32be3a4a54a4cb774d3abbcf04, hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e2b56dc34e5c4d9b9b417a77282b7051] into tmpdir=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp, totalSize=17.7 K 2024-11-26T23:28:36,189 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 27b1de05063140cf91e93de6c40568c2, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732663686014 2024-11-26T23:28:36,190 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e41dbc32be3a4a54a4cb774d3abbcf04, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=9, earliestPutTs=1732663696079 2024-11-26T23:28:36,190 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e2b56dc34e5c4d9b9b417a77282b7051, keycount=1, bloomtype=ROW, size=5.9 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732663706098 2024-11-26T23:28:36,201 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): e6b9d66717927e4826acc23a0a81da43#info#compaction#47 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:28:36,202 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/709ae765565d4b28a874144441cf2fef is 1080, key is row0001/info:/1732663686014/Put/seqid=0 2024-11-26T23:28:36,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741841_1017 (size=8296) 2024-11-26T23:28:36,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741841_1017 (size=8296) 2024-11-26T23:28:36,207 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:36,207 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:36,212 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/709ae765565d4b28a874144441cf2fef as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/709ae765565d4b28a874144441cf2fef 2024-11-26T23:28:36,218 INFO [Time-limited test {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e6b9d66717927e4826acc23a0a81da43/info of e6b9d66717927e4826acc23a0a81da43 into 709ae765565d4b28a874144441cf2fef(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:28:36,218 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e6b9d66717927e4826acc23a0a81da43: 2024-11-26T23:28:36,220 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C45123%2C1732663675044.1732663716220 2024-11-26T23:28:36,226 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:36,226 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:36,226 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:36,226 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:36,226 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:36,226 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663706100 with entries=4, filesize=2.45 KB; new WAL /user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663716220 2024-11-26T23:28:36,227 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39003:39003),(127.0.0.1/127.0.0.1:34981:34981)] 2024-11-26T23:28:36,227 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663706100 is not closed yet, will try archiving it next time 2024-11-26T23:28:36,227 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663675436 to hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/oldWALs/c707e9ace675%2C45123%2C1732663675044.1732663675436 2024-11-26T23:28:36,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741839_1015 (size=2520) 2024-11-26T23:28:36,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:28:36,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741839_1015 (size=2520) 2024-11-26T23:28:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:28:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T23:28:36,231 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_PREPARE 2024-11-26T23:28:36,231 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-26T23:28:36,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-26T23:28:36,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45123 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-26T23:28:36,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:36,385 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing e6b9d66717927e4826acc23a0a81da43 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-26T23:28:36,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/60cd5f90db444f53a1e93c68eb0ae6d3 is 1080, key is row0000/info:/1732663716219/Put/seqid=0 2024-11-26T23:28:36,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741843_1019 (size=6033) 2024-11-26T23:28:36,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741843_1019 (size=6033) 2024-11-26T23:28:36,394 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/60cd5f90db444f53a1e93c68eb0ae6d3 2024-11-26T23:28:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/60cd5f90db444f53a1e93c68eb0ae6d3 as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/60cd5f90db444f53a1e93c68eb0ae6d3 2024-11-26T23:28:36,404 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/60cd5f90db444f53a1e93c68eb0ae6d3, entries=1, sequenceid=18, filesize=5.9 K 2024-11-26T23:28:36,405 INFO [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e6b9d66717927e4826acc23a0a81da43 in 20ms, sequenceid=18, compaction requested=false 2024-11-26T23:28:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for e6b9d66717927e4826acc23a0a81da43: 2024-11-26T23:28:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:36,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-26T23:28:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-26T23:28:36,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-26T23:28:36,410 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 175 msec 2024-11-26T23:28:36,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestLogRolling-testCompactionRecordDoesntBlockRolling in 183 msec 2024-11-26T23:28:36,464 INFO [master/c707e9ace675:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T23:28:36,464 INFO [master/c707e9ace675:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T23:28:37,208 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:37,208 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:38,209 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:38,209 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:39,209 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:39,209 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:40,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:40,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:41,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:41,210 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:41,315 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region e6b9d66717927e4826acc23a0a81da43, had cached 0 bytes from a total of 14329 2024-11-26T23:28:42,211 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:42,211 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:43,212 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:43,212 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:44,212 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:44,212 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:45,213 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:45,213 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:46,213 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:46,213 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-26T23:28:46,257 INFO [RPCClient-NioEventLoopGroup-4-3 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestLogRolling-testCompactionRecordDoesntBlockRolling completed 2024-11-26T23:28:46,260 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C45123%2C1732663675044.1732663726259 2024-11-26T23:28:46,266 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,266 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,266 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,266 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,266 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,266 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663716220 with entries=3, filesize=1.97 KB; new WAL /user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663726259 2024-11-26T23:28:46,267 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39003:39003),(127.0.0.1/127.0.0.1:34981:34981)] 2024-11-26T23:28:46,267 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663716220 is not closed yet, will try archiving it next time 2024-11-26T23:28:46,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T23:28:46,267 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/WALs/c707e9ace675,45123,1732663675044/c707e9ace675%2C45123%2C1732663675044.1732663706100 to hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/oldWALs/c707e9ace675%2C45123%2C1732663675044.1732663706100 2024-11-26T23:28:46,267 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:28:46,268 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:28:46,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:28:46,268 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:28:46,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741842_1018 (size=2026) 2024-11-26T23:28:46,268 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T23:28:46,268 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1734479598, stopped=false 2024-11-26T23:28:46,268 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c707e9ace675,34975,1732663674994 2024-11-26T23:28:46,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741842_1018 (size=2026) 2024-11-26T23:28:46,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:28:46,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:28:46,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:28:46,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:46,270 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:46,270 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:28:46,270 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:28:46,271 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:28:46,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:28:46,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:28:46,271 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,45123,1732663675044' ***** 2024-11-26T23:28:46,271 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T23:28:46,271 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:28:46,271 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(878): Closing user regions 2024-11-26T23:28:46,271 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(3091): Received CLOSE for e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:28:46,272 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e6b9d66717927e4826acc23a0a81da43, disabling compactions & flushes 2024-11-26T23:28:46,272 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:46,272 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:46,272 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. after waiting 0 ms 2024-11-26T23:28:46,272 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:46,272 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing e6b9d66717927e4826acc23a0a81da43 1/1 column families, dataSize=1.05 KB heapSize=1.38 KB 2024-11-26T23:28:46,273 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:28:46,273 INFO [RS:0;c707e9ace675:45123 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T23:28:46,273 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:28:46,273 INFO [RS:0;c707e9ace675:45123 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T23:28:46,273 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(959): stopping server c707e9ace675,45123,1732663675044 2024-11-26T23:28:46,273 INFO [RS:0;c707e9ace675:45123 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:28:46,273 INFO [RS:0;c707e9ace675:45123 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c707e9ace675:45123. 2024-11-26T23:28:46,273 DEBUG [RS:0;c707e9ace675:45123 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:28:46,273 DEBUG [RS:0;c707e9ace675:45123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:28:46,273 INFO [RS:0;c707e9ace675:45123 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:28:46,273 INFO [RS:0;c707e9ace675:45123 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:28:46,273 INFO [RS:0;c707e9ace675:45123 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:28:46,273 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T23:28:46,278 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-26T23:28:46,278 DEBUG [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(1325): Online Regions={e6b9d66717927e4826acc23a0a81da43=TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43., 1588230740=hbase:meta,,1.1588230740} 2024-11-26T23:28:46,278 DEBUG [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, e6b9d66717927e4826acc23a0a81da43 2024-11-26T23:28:46,278 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:28:46,278 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:28:46,278 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:28:46,278 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:28:46,278 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:28:46,278 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.89 KB heapSize=3.91 KB 2024-11-26T23:28:46,280 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/7e578a5285814e6c9971dabe1c865356 is 1080, key is row0001/info:/1732663726258/Put/seqid=0 2024-11-26T23:28:46,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741845_1021 (size=6033) 2024-11-26T23:28:46,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741845_1021 (size=6033) 2024-11-26T23:28:46,288 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.05 KB at sequenceid=22 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/7e578a5285814e6c9971dabe1c865356 2024-11-26T23:28:46,293 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/.tmp/info/7e578a5285814e6c9971dabe1c865356 as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/7e578a5285814e6c9971dabe1c865356 2024-11-26T23:28:46,297 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/7e578a5285814e6c9971dabe1c865356, entries=1, sequenceid=22, filesize=5.9 K 2024-11-26T23:28:46,298 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e6b9d66717927e4826acc23a0a81da43 in 26ms, sequenceid=22, compaction requested=true 2024-11-26T23:28:46,299 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/27b1de05063140cf91e93de6c40568c2, hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e41dbc32be3a4a54a4cb774d3abbcf04, hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e2b56dc34e5c4d9b9b417a77282b7051] to archive 2024-11-26T23:28:46,300 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:28:46,301 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/27b1de05063140cf91e93de6c40568c2 to hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/27b1de05063140cf91e93de6c40568c2 2024-11-26T23:28:46,302 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e41dbc32be3a4a54a4cb774d3abbcf04 to hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e41dbc32be3a4a54a4cb774d3abbcf04 2024-11-26T23:28:46,303 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e2b56dc34e5c4d9b9b417a77282b7051 to hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/archive/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/info/e2b56dc34e5c4d9b9b417a77282b7051 2024-11-26T23:28:46,304 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/.tmp/info/3c72a53fdf424d0785b7c35bd3687f5d is 227, key is TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43./info:regioninfo/1732663676328/Put/seqid=0 2024-11-26T23:28:46,304 DEBUG [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c707e9ace675:34975 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-26T23:28:46,304 WARN [StoreCloser-TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [27b1de05063140cf91e93de6c40568c2=6033, e41dbc32be3a4a54a4cb774d3abbcf04=6033, e2b56dc34e5c4d9b9b417a77282b7051=6033] 2024-11-26T23:28:46,307 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-26T23:28:46,307 INFO [regionserver/c707e9ace675:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-26T23:28:46,309 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/default/TestLogRolling-testCompactionRecordDoesntBlockRolling/e6b9d66717927e4826acc23a0a81da43/recovered.edits/25.seqid, newMaxSeqId=25, maxSeqId=1 2024-11-26T23:28:46,309 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:46,309 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e6b9d66717927e4826acc23a0a81da43: Waiting for close lock at 1732663726272Running coprocessor pre-close hooks at 1732663726272Disabling compacts and flushes for region at 1732663726272Disabling writes for close at 1732663726272Obtaining lock to block concurrent updates at 1732663726272Preparing flush snapshotting stores in e6b9d66717927e4826acc23a0a81da43 at 1732663726272Finished memstore snapshotting TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43., syncing WAL and waiting on mvcc, flushsize=dataSize=1076, getHeapSize=1392, getOffHeapSize=0, getCellsCount=1 at 1732663726272Flushing stores of TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. at 1732663726273 (+1 ms)Flushing e6b9d66717927e4826acc23a0a81da43/info: creating writer at 1732663726273Flushing e6b9d66717927e4826acc23a0a81da43/info: appending metadata at 1732663726280 (+7 ms)Flushing e6b9d66717927e4826acc23a0a81da43/info: closing flushed file at 1732663726280Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@128b852c: reopening flushed file at 1732663726293 (+13 ms)Finished flush of dataSize ~1.05 KB/1076, heapSize ~1.36 KB/1392, currentSize=0 B/0 for e6b9d66717927e4826acc23a0a81da43 in 26ms, sequenceid=22, compaction requested=true at 1732663726298 (+5 ms)Writing region close event to WAL at 1732663726304 (+6 ms)Running coprocessor post-close hooks at 1732663726309 (+5 ms)Closed at 1732663726309 2024-11-26T23:28:46,310 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testCompactionRecordDoesntBlockRolling,,1732663675975.e6b9d66717927e4826acc23a0a81da43. 2024-11-26T23:28:46,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741846_1022 (size=7308) 2024-11-26T23:28:46,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741846_1022 (size=7308) 2024-11-26T23:28:46,312 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/.tmp/info/3c72a53fdf424d0785b7c35bd3687f5d 2024-11-26T23:28:46,337 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/.tmp/ns/c9fad397b5a9440ba19113a03fd32e96 is 43, key is default/ns:d/1732663675854/Put/seqid=0 2024-11-26T23:28:46,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741847_1023 (size=5153) 2024-11-26T23:28:46,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741847_1023 (size=5153) 2024-11-26T23:28:46,342 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/.tmp/ns/c9fad397b5a9440ba19113a03fd32e96 2024-11-26T23:28:46,361 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/.tmp/table/b4fc8ce7f5fe44eda1979839c564b495 is 89, key is TestLogRolling-testCompactionRecordDoesntBlockRolling/table:state/1732663676337/Put/seqid=0 2024-11-26T23:28:46,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741848_1024 (size=5508) 2024-11-26T23:28:46,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741848_1024 (size=5508) 2024-11-26T23:28:46,367 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=170 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/.tmp/table/b4fc8ce7f5fe44eda1979839c564b495 2024-11-26T23:28:46,372 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/.tmp/info/3c72a53fdf424d0785b7c35bd3687f5d as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/info/3c72a53fdf424d0785b7c35bd3687f5d 2024-11-26T23:28:46,377 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/info/3c72a53fdf424d0785b7c35bd3687f5d, entries=10, sequenceid=11, filesize=7.1 K 2024-11-26T23:28:46,377 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/.tmp/ns/c9fad397b5a9440ba19113a03fd32e96 as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/ns/c9fad397b5a9440ba19113a03fd32e96 2024-11-26T23:28:46,382 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/ns/c9fad397b5a9440ba19113a03fd32e96, entries=2, sequenceid=11, filesize=5.0 K 2024-11-26T23:28:46,383 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/.tmp/table/b4fc8ce7f5fe44eda1979839c564b495 as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/table/b4fc8ce7f5fe44eda1979839c564b495 2024-11-26T23:28:46,387 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/table/b4fc8ce7f5fe44eda1979839c564b495, entries=2, sequenceid=11, filesize=5.4 K 2024-11-26T23:28:46,388 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=11, compaction requested=false 2024-11-26T23:28:46,392 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-11-26T23:28:46,393 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:28:46,393 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:28:46,393 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663726278Running coprocessor pre-close hooks at 1732663726278Disabling compacts and flushes for region at 1732663726278Disabling writes for close at 1732663726278Obtaining lock to block concurrent updates at 1732663726278Preparing flush snapshotting stores in 1588230740 at 1732663726278Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1932, getHeapSize=3936, getOffHeapSize=0, getCellsCount=14 at 1732663726279 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732663726279Flushing 1588230740/info: creating writer at 1732663726279Flushing 1588230740/info: appending metadata at 1732663726303 (+24 ms)Flushing 1588230740/info: closing flushed file at 1732663726303Flushing 1588230740/ns: creating writer at 1732663726318 (+15 ms)Flushing 1588230740/ns: appending metadata at 1732663726336 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1732663726336Flushing 1588230740/table: creating writer at 1732663726347 (+11 ms)Flushing 1588230740/table: appending metadata at 1732663726361 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732663726361Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50cc9728: reopening flushed file at 1732663726371 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a90eaee: reopening flushed file at 1732663726377 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a77673f: reopening flushed file at 1732663726382 (+5 ms)Finished flush of dataSize ~1.89 KB/1932, heapSize ~3.61 KB/3696, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=11, compaction requested=false at 1732663726388 (+6 ms)Writing region close event to WAL at 1732663726389 (+1 ms)Running coprocessor post-close hooks at 1732663726393 (+4 ms)Closed at 1732663726393 2024-11-26T23:28:46,393 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T23:28:46,478 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,45123,1732663675044; all regions closed. 2024-11-26T23:28:46,479 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,479 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,479 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,479 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,479 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741834_1010 (size=3306) 2024-11-26T23:28:46,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741834_1010 (size=3306) 2024-11-26T23:28:46,484 DEBUG [RS:0;c707e9ace675:45123 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/oldWALs 2024-11-26T23:28:46,484 INFO [RS:0;c707e9ace675:45123 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C45123%2C1732663675044.meta:.meta(num 1732663675801) 2024-11-26T23:28:46,484 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,484 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,484 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,484 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,484 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741844_1020 (size=1252) 2024-11-26T23:28:46,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741844_1020 (size=1252) 2024-11-26T23:28:46,489 DEBUG [RS:0;c707e9ace675:45123 {}] wal.AbstractFSWAL(1256): Moved 2 WAL file(s) to /user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/oldWALs 2024-11-26T23:28:46,489 INFO [RS:0;c707e9ace675:45123 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C45123%2C1732663675044:(num 1732663726259) 2024-11-26T23:28:46,489 DEBUG [RS:0;c707e9ace675:45123 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:28:46,489 INFO [RS:0;c707e9ace675:45123 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:28:46,489 INFO [RS:0;c707e9ace675:45123 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:28:46,489 INFO [RS:0;c707e9ace675:45123 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T23:28:46,489 INFO [RS:0;c707e9ace675:45123 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:28:46,489 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:28:46,490 INFO [RS:0;c707e9ace675:45123 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45123 2024-11-26T23:28:46,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:28:46,492 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,45123,1732663675044 2024-11-26T23:28:46,492 INFO [RS:0;c707e9ace675:45123 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:28:46,493 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,45123,1732663675044] 2024-11-26T23:28:46,495 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,45123,1732663675044 already deleted, retry=false 2024-11-26T23:28:46,495 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c707e9ace675,45123,1732663675044 expired; onlineServers=0 2024-11-26T23:28:46,495 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c707e9ace675,34975,1732663674994' ***** 2024-11-26T23:28:46,495 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T23:28:46,495 INFO [M:0;c707e9ace675:34975 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:28:46,495 INFO [M:0;c707e9ace675:34975 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:28:46,495 DEBUG [M:0;c707e9ace675:34975 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T23:28:46,496 DEBUG [M:0;c707e9ace675:34975 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T23:28:46,496 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T23:28:46,496 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663675198 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663675198,5,FailOnTimeoutGroup] 2024-11-26T23:28:46,496 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663675199 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663675199,5,FailOnTimeoutGroup] 2024-11-26T23:28:46,496 INFO [M:0;c707e9ace675:34975 {}] hbase.ChoreService(370): Chore service for: master/c707e9ace675:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T23:28:46,496 INFO [M:0;c707e9ace675:34975 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:28:46,496 DEBUG [M:0;c707e9ace675:34975 {}] master.HMaster(1795): Stopping service threads 2024-11-26T23:28:46,496 INFO [M:0;c707e9ace675:34975 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T23:28:46,496 INFO [M:0;c707e9ace675:34975 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:28:46,496 INFO [M:0;c707e9ace675:34975 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T23:28:46,496 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T23:28:46,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T23:28:46,497 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:46,497 DEBUG [M:0;c707e9ace675:34975 {}] zookeeper.ZKUtil(347): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T23:28:46,497 WARN [M:0;c707e9ace675:34975 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T23:28:46,498 INFO [M:0;c707e9ace675:34975 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/.lastflushedseqids 2024-11-26T23:28:46,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741849_1025 (size=130) 2024-11-26T23:28:46,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741849_1025 (size=130) 2024-11-26T23:28:46,503 INFO [M:0;c707e9ace675:34975 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T23:28:46,504 INFO [M:0;c707e9ace675:34975 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T23:28:46,504 DEBUG [M:0;c707e9ace675:34975 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:28:46,504 INFO [M:0;c707e9ace675:34975 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:28:46,504 DEBUG [M:0;c707e9ace675:34975 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:28:46,504 DEBUG [M:0;c707e9ace675:34975 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:28:46,504 DEBUG [M:0;c707e9ace675:34975 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:28:46,504 INFO [M:0;c707e9ace675:34975 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=43.61 KB heapSize=55.02 KB 2024-11-26T23:28:46,520 DEBUG [M:0;c707e9ace675:34975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4360c976a6784146a40a1269ee6d82b7 is 82, key is hbase:meta,,1/info:regioninfo/1732663675834/Put/seqid=0 2024-11-26T23:28:46,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741850_1026 (size=5672) 2024-11-26T23:28:46,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741850_1026 (size=5672) 2024-11-26T23:28:46,525 INFO [M:0;c707e9ace675:34975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4360c976a6784146a40a1269ee6d82b7 2024-11-26T23:28:46,552 DEBUG [M:0;c707e9ace675:34975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c0523e3f8c97424ea987f8af92a830bf is 799, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732663676341/Put/seqid=0 2024-11-26T23:28:46,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741851_1027 (size=7825) 2024-11-26T23:28:46,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741851_1027 (size=7825) 2024-11-26T23:28:46,558 INFO [M:0;c707e9ace675:34975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.01 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c0523e3f8c97424ea987f8af92a830bf 2024-11-26T23:28:46,563 INFO [M:0;c707e9ace675:34975 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c0523e3f8c97424ea987f8af92a830bf 2024-11-26T23:28:46,578 DEBUG [M:0;c707e9ace675:34975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3ec5a8b87654bd98f8b184967ba0a12 is 69, key is c707e9ace675,45123,1732663675044/rs:state/1732663675278/Put/seqid=0 2024-11-26T23:28:46,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741852_1028 (size=5156) 2024-11-26T23:28:46,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741852_1028 (size=5156) 2024-11-26T23:28:46,586 INFO [M:0;c707e9ace675:34975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3ec5a8b87654bd98f8b184967ba0a12 2024-11-26T23:28:46,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:28:46,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45123-0x101569b72f40001, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:28:46,593 INFO [RS:0;c707e9ace675:45123 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:28:46,594 INFO [RS:0;c707e9ace675:45123 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,45123,1732663675044; zookeeper connection closed. 2024-11-26T23:28:46,594 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@63369f59 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@63369f59 2024-11-26T23:28:46,594 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T23:28:46,605 DEBUG [M:0;c707e9ace675:34975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/405f359526254376b4c0e2bc767da1ab is 52, key is load_balancer_on/state:d/1732663675971/Put/seqid=0 2024-11-26T23:28:46,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741853_1029 (size=5056) 2024-11-26T23:28:46,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741853_1029 (size=5056) 2024-11-26T23:28:46,611 INFO [M:0;c707e9ace675:34975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/405f359526254376b4c0e2bc767da1ab 2024-11-26T23:28:46,616 DEBUG [M:0;c707e9ace675:34975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/4360c976a6784146a40a1269ee6d82b7 as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4360c976a6784146a40a1269ee6d82b7 2024-11-26T23:28:46,621 INFO [M:0;c707e9ace675:34975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/4360c976a6784146a40a1269ee6d82b7, entries=8, sequenceid=121, filesize=5.5 K 2024-11-26T23:28:46,622 DEBUG [M:0;c707e9ace675:34975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/c0523e3f8c97424ea987f8af92a830bf as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c0523e3f8c97424ea987f8af92a830bf 2024-11-26T23:28:46,627 INFO [M:0;c707e9ace675:34975 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c0523e3f8c97424ea987f8af92a830bf 2024-11-26T23:28:46,627 INFO [M:0;c707e9ace675:34975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/c0523e3f8c97424ea987f8af92a830bf, entries=14, sequenceid=121, filesize=7.6 K 2024-11-26T23:28:46,628 DEBUG [M:0;c707e9ace675:34975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a3ec5a8b87654bd98f8b184967ba0a12 as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a3ec5a8b87654bd98f8b184967ba0a12 2024-11-26T23:28:46,633 INFO [M:0;c707e9ace675:34975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a3ec5a8b87654bd98f8b184967ba0a12, entries=1, sequenceid=121, filesize=5.0 K 2024-11-26T23:28:46,634 DEBUG [M:0;c707e9ace675:34975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/405f359526254376b4c0e2bc767da1ab as hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/405f359526254376b4c0e2bc767da1ab 2024-11-26T23:28:46,639 INFO [M:0;c707e9ace675:34975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39031/user/jenkins/test-data/9b9bb425-1c8c-0fce-8df9-9a5f9953f9de/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/405f359526254376b4c0e2bc767da1ab, entries=1, sequenceid=121, filesize=4.9 K 2024-11-26T23:28:46,640 INFO [M:0;c707e9ace675:34975 {}] regionserver.HRegion(3140): Finished flush of dataSize ~43.61 KB/44659, heapSize ~54.95 KB/56272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=121, compaction requested=false 2024-11-26T23:28:46,642 INFO [M:0;c707e9ace675:34975 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:28:46,642 DEBUG [M:0;c707e9ace675:34975 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663726504Disabling compacts and flushes for region at 1732663726504Disabling writes for close at 1732663726504Obtaining lock to block concurrent updates at 1732663726504Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732663726504Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=44659, getHeapSize=56272, getOffHeapSize=0, getCellsCount=140 at 1732663726504Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732663726505 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732663726505Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732663726519 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732663726519Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732663726530 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732663726551 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732663726552 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732663726563 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732663726578 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732663726578Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732663726591 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732663726605 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732663726605Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@333e6b48: reopening flushed file at 1732663726616 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35a8697e: reopening flushed file at 1732663726622 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@46979f9a: reopening flushed file at 1732663726628 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5162a253: reopening flushed file at 1732663726633 (+5 ms)Finished flush of dataSize ~43.61 KB/44659, heapSize ~54.95 KB/56272, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 136ms, sequenceid=121, compaction requested=false at 1732663726640 (+7 ms)Writing region close event to WAL at 1732663726642 (+2 ms)Closed at 1732663726642 2024-11-26T23:28:46,642 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,642 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,643 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,643 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,643 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:28:46,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36941 is added to blk_1073741830_1006 (size=53056) 2024-11-26T23:28:46,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41663 is added to blk_1073741830_1006 (size=53056) 2024-11-26T23:28:46,646 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:28:46,646 INFO [M:0;c707e9ace675:34975 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T23:28:46,646 INFO [M:0;c707e9ace675:34975 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34975 2024-11-26T23:28:46,646 INFO [M:0;c707e9ace675:34975 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:28:46,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:28:46,748 INFO [M:0;c707e9ace675:34975 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:28:46,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34975-0x101569b72f40000, quorum=127.0.0.1:58917, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:28:46,750 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2afda299{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:28:46,751 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6bd1231f{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:28:46,751 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:28:46,751 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b5fa12b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:28:46,751 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3838d9cb{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/hadoop.log.dir/,STOPPED} 2024-11-26T23:28:46,752 WARN [BP-2058959891-172.17.0.2-1732663674339 heartbeating to localhost/127.0.0.1:39031 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:28:46,752 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:28:46,752 WARN [BP-2058959891-172.17.0.2-1732663674339 heartbeating to localhost/127.0.0.1:39031 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2058959891-172.17.0.2-1732663674339 (Datanode Uuid 59a8e7b1-b5d3-4838-9820-62bf9620f169) service to localhost/127.0.0.1:39031 2024-11-26T23:28:46,753 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:28:46,753 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d/data/data3/current/BP-2058959891-172.17.0.2-1732663674339 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:28:46,753 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d/data/data4/current/BP-2058959891-172.17.0.2-1732663674339 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:28:46,754 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:28:46,755 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@5ea88c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:28:46,755 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@78617008{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:28:46,756 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:28:46,756 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e7fac47{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:28:46,756 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@371e2711{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/hadoop.log.dir/,STOPPED} 2024-11-26T23:28:46,757 WARN [BP-2058959891-172.17.0.2-1732663674339 heartbeating to localhost/127.0.0.1:39031 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:28:46,757 WARN [BP-2058959891-172.17.0.2-1732663674339 heartbeating to localhost/127.0.0.1:39031 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-2058959891-172.17.0.2-1732663674339 (Datanode Uuid d10d802c-e6f2-4b8f-ac38-8593ecb4d4d2) service to localhost/127.0.0.1:39031 2024-11-26T23:28:46,757 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:28:46,757 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:28:46,758 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d/data/data1/current/BP-2058959891-172.17.0.2-1732663674339 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:28:46,758 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/cluster_fa885bdf-ce5a-f062-9e15-46732ec0721d/data/data2/current/BP-2058959891-172.17.0.2-1732663674339 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:28:46,758 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:28:46,764 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4682d650{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:28:46,765 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@42501282{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:28:46,765 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:28:46,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@701842fe{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:28:46,765 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@56e526c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/hadoop.log.dir/,STOPPED} 2024-11-26T23:28:46,771 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T23:28:46,788 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T23:28:46,796 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testCompactionRecordDoesntBlockRolling Thread=206 (was 179) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39031 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-35-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39031 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: regionserver/c707e9ace675:0.leaseChecker java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) Potentially hanging thread: nioEventLoopGroup-35-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39031 from jenkins.hfs.5 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: HMaster-EventLoopGroup-12-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39031 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-37-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:39031 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.5@localhost:39031 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-36-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:39031 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-36-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-35-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-34-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-13-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39031 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-12-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:39031 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=483 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=57 (was 116), ProcessCount=11 (was 11), AvailableMemoryMB=6635 (was 6717) 2024-11-26T23:28:46,804 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRolling Thread=206, OpenFileDescriptor=483, MaxFileDescriptor=1048576, SystemLoadAverage=57, ProcessCount=11, AvailableMemoryMB=6637 2024-11-26T23:28:46,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T23:28:46,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/hadoop.log.dir so I do NOT create it in target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95 2024-11-26T23:28:46,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/3b43347a-70d3-e9e7-1eba-cc74ae14f093/hadoop.tmp.dir so I do NOT create it in target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95 2024-11-26T23:28:46,804 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4, deleteOnExit=true 2024-11-26T23:28:46,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/test.cache.data in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/hadoop.log.dir in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T23:28:46,805 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:28:46,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T23:28:46,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/nfs.dump.dir in system properties and HBase conf 2024-11-26T23:28:46,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/java.io.tmpdir in system properties and HBase conf 2024-11-26T23:28:46,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:28:46,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T23:28:46,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T23:28:46,819 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:28:46,892 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:28:46,895 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:28:46,896 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:28:46,896 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:28:46,896 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:28:46,897 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:28:46,897 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@62c26df6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:28:46,898 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@188ddc10{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:28:47,012 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6e1ebd6b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/java.io.tmpdir/jetty-localhost-35737-hadoop-hdfs-3_4_1-tests_jar-_-any-6500939927668529416/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:28:47,013 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1a0fc9ad{HTTP/1.1, (http/1.1)}{localhost:35737} 2024-11-26T23:28:47,013 INFO [Time-limited test {}] server.Server(415): Started @236358ms 2024-11-26T23:28:47,026 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:28:47,093 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:28:47,096 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:28:47,096 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:28:47,096 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:28:47,096 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:28:47,097 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2d8c7847{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:28:47,097 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595aaa8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:28:47,210 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@26881465{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/java.io.tmpdir/jetty-localhost-42557-hadoop-hdfs-3_4_1-tests_jar-_-any-10078270833915984034/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:28:47,210 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5ecf816b{HTTP/1.1, (http/1.1)}{localhost:42557} 2024-11-26T23:28:47,210 INFO [Time-limited test {}] server.Server(415): Started @236555ms 2024-11-26T23:28:47,212 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:28:47,214 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:47,214 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:47,240 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:28:47,243 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:28:47,244 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:28:47,244 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:28:47,244 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:28:47,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5689196f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:28:47,244 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3612be31{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:28:47,293 INFO [regionserver/c707e9ace675:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:28:47,311 WARN [Thread-1968 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4/data/data2/current/BP-893640581-172.17.0.2-1732663726836/current, will proceed with Du for space computation calculation, 2024-11-26T23:28:47,311 WARN [Thread-1967 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4/data/data1/current/BP-893640581-172.17.0.2-1732663726836/current, will proceed with Du for space computation calculation, 2024-11-26T23:28:47,332 WARN [Thread-1946 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:28:47,334 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d28cfb2f4e7359 with lease ID 0x34ea1e3b9dfa5219: Processing first storage report for DS-5b6ca9f8-4f03-495d-976e-18720f840730 from datanode DatanodeRegistration(127.0.0.1:33689, datanodeUuid=38d37e44-0c61-4580-8e9e-599b5e47f98d, infoPort=34831, infoSecurePort=0, ipcPort=33241, storageInfo=lv=-57;cid=testClusterID;nsid=284314398;c=1732663726836) 2024-11-26T23:28:47,335 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d28cfb2f4e7359 with lease ID 0x34ea1e3b9dfa5219: from storage DS-5b6ca9f8-4f03-495d-976e-18720f840730 node DatanodeRegistration(127.0.0.1:33689, datanodeUuid=38d37e44-0c61-4580-8e9e-599b5e47f98d, infoPort=34831, infoSecurePort=0, ipcPort=33241, storageInfo=lv=-57;cid=testClusterID;nsid=284314398;c=1732663726836), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:28:47,335 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5d28cfb2f4e7359 with lease ID 0x34ea1e3b9dfa5219: Processing first storage report for DS-8f360320-a3f7-4970-943d-1fc24c018069 from datanode DatanodeRegistration(127.0.0.1:33689, datanodeUuid=38d37e44-0c61-4580-8e9e-599b5e47f98d, infoPort=34831, infoSecurePort=0, ipcPort=33241, storageInfo=lv=-57;cid=testClusterID;nsid=284314398;c=1732663726836) 2024-11-26T23:28:47,335 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5d28cfb2f4e7359 with lease ID 0x34ea1e3b9dfa5219: from storage DS-8f360320-a3f7-4970-943d-1fc24c018069 node DatanodeRegistration(127.0.0.1:33689, datanodeUuid=38d37e44-0c61-4580-8e9e-599b5e47f98d, infoPort=34831, infoSecurePort=0, ipcPort=33241, storageInfo=lv=-57;cid=testClusterID;nsid=284314398;c=1732663726836), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:28:47,374 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@28377467{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/java.io.tmpdir/jetty-localhost-35799-hadoop-hdfs-3_4_1-tests_jar-_-any-8400165754305016068/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:28:47,374 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@41a74ab6{HTTP/1.1, (http/1.1)}{localhost:35799} 2024-11-26T23:28:47,374 INFO [Time-limited test {}] server.Server(415): Started @236719ms 2024-11-26T23:28:47,375 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:28:47,477 WARN [Thread-1993 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4/data/data3/current/BP-893640581-172.17.0.2-1732663726836/current, will proceed with Du for space computation calculation, 2024-11-26T23:28:47,478 WARN [Thread-1994 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4/data/data4/current/BP-893640581-172.17.0.2-1732663726836/current, will proceed with Du for space computation calculation, 2024-11-26T23:28:47,494 WARN [Thread-1982 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:28:47,496 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1592928286af869 with lease ID 0x34ea1e3b9dfa521a: Processing first storage report for DS-0fa6bb87-789e-4ef7-9a2b-345e4957e50d from datanode DatanodeRegistration(127.0.0.1:43015, datanodeUuid=e6138815-011f-4ed6-88b6-260564e409ce, infoPort=39207, infoSecurePort=0, ipcPort=33497, storageInfo=lv=-57;cid=testClusterID;nsid=284314398;c=1732663726836) 2024-11-26T23:28:47,496 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1592928286af869 with lease ID 0x34ea1e3b9dfa521a: from storage DS-0fa6bb87-789e-4ef7-9a2b-345e4957e50d node DatanodeRegistration(127.0.0.1:43015, datanodeUuid=e6138815-011f-4ed6-88b6-260564e409ce, infoPort=39207, infoSecurePort=0, ipcPort=33497, storageInfo=lv=-57;cid=testClusterID;nsid=284314398;c=1732663726836), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:28:47,496 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe1592928286af869 with lease ID 0x34ea1e3b9dfa521a: Processing first storage report for DS-2cd80f70-cffb-4776-832a-9ffa491d3342 from datanode DatanodeRegistration(127.0.0.1:43015, datanodeUuid=e6138815-011f-4ed6-88b6-260564e409ce, infoPort=39207, infoSecurePort=0, ipcPort=33497, storageInfo=lv=-57;cid=testClusterID;nsid=284314398;c=1732663726836) 2024-11-26T23:28:47,496 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe1592928286af869 with lease ID 0x34ea1e3b9dfa521a: from storage DS-2cd80f70-cffb-4776-832a-9ffa491d3342 node DatanodeRegistration(127.0.0.1:43015, datanodeUuid=e6138815-011f-4ed6-88b6-260564e409ce, infoPort=39207, infoSecurePort=0, ipcPort=33497, storageInfo=lv=-57;cid=testClusterID;nsid=284314398;c=1732663726836), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:28:47,497 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95 2024-11-26T23:28:47,499 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4/zookeeper_0, clientPort=54328, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T23:28:47,500 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54328 2024-11-26T23:28:47,501 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:28:47,502 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:28:47,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:28:47,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:28:47,514 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1 with version=8 2024-11-26T23:28:47,514 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/hbase-staging 2024-11-26T23:28:47,516 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:28:47,516 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:28:47,516 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:28:47,516 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:28:47,516 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:28:47,516 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:28:47,516 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T23:28:47,516 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:28:47,517 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42975 2024-11-26T23:28:47,518 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42975 connecting to ZooKeeper ensemble=127.0.0.1:54328 2024-11-26T23:28:47,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:429750x0, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:28:47,531 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42975-0x101569c401a0000 connected 2024-11-26T23:28:47,549 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:28:47,551 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:28:47,552 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:28:47,553 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1, hbase.cluster.distributed=false 2024-11-26T23:28:47,554 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:28:47,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42975 2024-11-26T23:28:47,554 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42975 2024-11-26T23:28:47,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42975 2024-11-26T23:28:47,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42975 2024-11-26T23:28:47,555 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42975 2024-11-26T23:28:47,571 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:28:47,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:28:47,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:28:47,571 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:28:47,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:28:47,571 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:28:47,571 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:28:47,571 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:28:47,572 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38827 2024-11-26T23:28:47,573 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38827 connecting to ZooKeeper ensemble=127.0.0.1:54328 2024-11-26T23:28:47,573 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:28:47,575 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:28:47,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:388270x0, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:28:47,579 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:388270x0, quorum=127.0.0.1:54328, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:28:47,579 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38827-0x101569c401a0001 connected 2024-11-26T23:28:47,579 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:28:47,580 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:28:47,580 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T23:28:47,582 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:28:47,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38827 2024-11-26T23:28:47,583 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38827 2024-11-26T23:28:47,584 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38827 2024-11-26T23:28:47,584 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38827 2024-11-26T23:28:47,585 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38827 2024-11-26T23:28:47,596 DEBUG [M:0;c707e9ace675:42975 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c707e9ace675:42975 2024-11-26T23:28:47,597 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c707e9ace675,42975,1732663727516 2024-11-26T23:28:47,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:28:47,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:28:47,599 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c707e9ace675,42975,1732663727516 2024-11-26T23:28:47,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T23:28:47,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:47,600 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:47,600 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T23:28:47,601 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c707e9ace675,42975,1732663727516 from backup master directory 2024-11-26T23:28:47,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c707e9ace675,42975,1732663727516 2024-11-26T23:28:47,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:28:47,603 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:28:47,603 WARN [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:28:47,603 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c707e9ace675,42975,1732663727516 2024-11-26T23:28:47,606 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/hbase.id] with ID: 5aee37b8-58ee-4cb8-8e4d-f078267a7328 2024-11-26T23:28:47,606 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/.tmp/hbase.id 2024-11-26T23:28:47,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:28:47,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:28:47,613 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/.tmp/hbase.id]:[hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/hbase.id] 2024-11-26T23:28:47,625 INFO [master/c707e9ace675:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:28:47,625 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T23:28:47,626 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-26T23:28:47,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:47,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:47,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:28:47,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:28:47,658 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:28:47,659 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T23:28:47,659 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:28:47,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:28:47,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:28:47,670 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store 2024-11-26T23:28:47,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:28:47,676 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:28:47,676 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:28:47,677 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:28:47,677 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:28:47,677 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:28:47,677 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:28:47,677 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:28:47,677 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663727676Disabling compacts and flushes for region at 1732663727676Disabling writes for close at 1732663727677 (+1 ms)Writing region close event to WAL at 1732663727677Closed at 1732663727677 2024-11-26T23:28:47,677 WARN [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/.initializing 2024-11-26T23:28:47,677 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/WALs/c707e9ace675,42975,1732663727516 2024-11-26T23:28:47,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:28:47,680 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C42975%2C1732663727516, suffix=, logDir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/WALs/c707e9ace675,42975,1732663727516, archiveDir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/oldWALs, maxLogs=10 2024-11-26T23:28:47,680 INFO [master/c707e9ace675:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C42975%2C1732663727516.1732663727680 2024-11-26T23:28:47,684 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/WALs/c707e9ace675,42975,1732663727516/c707e9ace675%2C42975%2C1732663727516.1732663727680 2024-11-26T23:28:47,688 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34831:34831),(127.0.0.1/127.0.0.1:39207:39207)] 2024-11-26T23:28:47,691 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:28:47,691 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:28:47,691 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,691 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,692 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,694 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T23:28:47,694 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:47,694 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:28:47,694 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,695 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T23:28:47,695 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:47,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:28:47,696 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T23:28:47,697 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:47,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:28:47,697 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,698 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T23:28:47,699 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:47,699 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:28:47,699 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,700 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,700 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,701 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,701 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,701 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:28:47,702 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:28:47,704 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:28:47,705 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=859370, jitterRate=0.09274645149707794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:28:47,705 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732663727691Initializing all the Stores at 1732663727692 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663727692Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663727692Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663727692Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663727692Cleaning up temporary data from old regions at 1732663727701 (+9 ms)Region opened successfully at 1732663727705 (+4 ms) 2024-11-26T23:28:47,705 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T23:28:47,708 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba584a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:28:47,709 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T23:28:47,709 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T23:28:47,709 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T23:28:47,709 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T23:28:47,710 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T23:28:47,710 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-26T23:28:47,710 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T23:28:47,712 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T23:28:47,712 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T23:28:47,714 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T23:28:47,714 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T23:28:47,714 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T23:28:47,715 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T23:28:47,715 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T23:28:47,716 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T23:28:47,718 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T23:28:47,719 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T23:28:47,720 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T23:28:47,722 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T23:28:47,723 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T23:28:47,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:28:47,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:28:47,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:47,724 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:47,725 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c707e9ace675,42975,1732663727516, sessionid=0x101569c401a0000, setting cluster-up flag (Was=false) 2024-11-26T23:28:47,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:47,728 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:47,733 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T23:28:47,734 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,42975,1732663727516 2024-11-26T23:28:47,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:47,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:47,748 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T23:28:47,749 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,42975,1732663727516 2024-11-26T23:28:47,750 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T23:28:47,752 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T23:28:47,752 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T23:28:47,752 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T23:28:47,752 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c707e9ace675,42975,1732663727516 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T23:28:47,753 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:28:47,753 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:28:47,753 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:28:47,753 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:28:47,753 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c707e9ace675:0, corePoolSize=10, maxPoolSize=10 2024-11-26T23:28:47,753 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,753 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:28:47,753 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,755 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732663757755 2024-11-26T23:28:47,755 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:28:47,755 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T23:28:47,755 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T23:28:47,755 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T23:28:47,755 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T23:28:47,755 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T23:28:47,755 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T23:28:47,755 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T23:28:47,756 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,756 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:47,756 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T23:28:47,756 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T23:28:47,756 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T23:28:47,756 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T23:28:47,757 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T23:28:47,757 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T23:28:47,757 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663727757,5,FailOnTimeoutGroup] 2024-11-26T23:28:47,757 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663727757,5,FailOnTimeoutGroup] 2024-11-26T23:28:47,758 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,758 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T23:28:47,758 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,758 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:28:47,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:28:47,764 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T23:28:47,765 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1 2024-11-26T23:28:47,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:28:47,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:28:47,773 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:28:47,774 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:28:47,775 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:28:47,775 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:47,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:28:47,776 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:28:47,777 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:28:47,777 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:47,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:28:47,777 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:28:47,778 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:28:47,778 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:47,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:28:47,779 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:28:47,779 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:28:47,780 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:47,780 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:28:47,780 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:28:47,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740 2024-11-26T23:28:47,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740 2024-11-26T23:28:47,782 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:28:47,782 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:28:47,783 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:28:47,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:28:47,785 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:28:47,786 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=840848, jitterRate=0.06919416785240173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:28:47,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732663727773Initializing all the Stores at 1732663727774 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663727774Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663727774Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663727774Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663727774Cleaning up temporary data from old regions at 1732663727782 (+8 ms)Region opened successfully at 1732663727786 (+4 ms) 2024-11-26T23:28:47,786 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(746): ClusterId : 5aee37b8-58ee-4cb8-8e4d-f078267a7328 2024-11-26T23:28:47,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:28:47,786 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:28:47,786 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:28:47,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:28:47,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:28:47,786 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:28:47,787 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:28:47,787 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663727786Disabling compacts and flushes for region at 1732663727786Disabling writes for close at 1732663727786Writing region close event to WAL at 1732663727787 (+1 ms)Closed at 1732663727787 2024-11-26T23:28:47,788 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:28:47,788 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T23:28:47,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T23:28:47,788 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:28:47,788 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:28:47,790 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:28:47,791 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T23:28:47,791 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:28:47,791 DEBUG [RS:0;c707e9ace675:38827 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b491c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:28:47,804 DEBUG [RS:0;c707e9ace675:38827 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c707e9ace675:38827 2024-11-26T23:28:47,804 INFO [RS:0;c707e9ace675:38827 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:28:47,804 INFO [RS:0;c707e9ace675:38827 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:28:47,804 DEBUG [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:28:47,805 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,42975,1732663727516 with port=38827, startcode=1732663727570 2024-11-26T23:28:47,805 DEBUG [RS:0;c707e9ace675:38827 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:28:47,807 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37509, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.6 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:28:47,807 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42975 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,38827,1732663727570 2024-11-26T23:28:47,807 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42975 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,38827,1732663727570 2024-11-26T23:28:47,809 DEBUG [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1 2024-11-26T23:28:47,809 DEBUG [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44869 2024-11-26T23:28:47,809 DEBUG [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:28:47,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:28:47,813 DEBUG [RS:0;c707e9ace675:38827 {}] zookeeper.ZKUtil(111): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,38827,1732663727570 2024-11-26T23:28:47,813 WARN [RS:0;c707e9ace675:38827 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:28:47,813 INFO [RS:0;c707e9ace675:38827 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:28:47,813 DEBUG [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570 2024-11-26T23:28:47,813 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,38827,1732663727570] 2024-11-26T23:28:47,818 INFO [RS:0;c707e9ace675:38827 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:28:47,819 INFO [RS:0;c707e9ace675:38827 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:28:47,820 INFO [RS:0;c707e9ace675:38827 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:28:47,820 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,820 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:28:47,821 INFO [RS:0;c707e9ace675:38827 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:28:47,821 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:28:47,821 DEBUG [RS:0;c707e9ace675:38827 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:28:47,822 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,822 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,822 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,822 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,822 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,822 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,38827,1732663727570-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:28:47,838 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:28:47,838 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,38827,1732663727570-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,838 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,838 INFO [RS:0;c707e9ace675:38827 {}] regionserver.Replication(171): c707e9ace675,38827,1732663727570 started 2024-11-26T23:28:47,852 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:47,852 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,38827,1732663727570, RpcServer on c707e9ace675/172.17.0.2:38827, sessionid=0x101569c401a0001 2024-11-26T23:28:47,852 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:28:47,852 DEBUG [RS:0;c707e9ace675:38827 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,38827,1732663727570 2024-11-26T23:28:47,852 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,38827,1732663727570' 2024-11-26T23:28:47,852 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:28:47,853 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:28:47,853 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:28:47,853 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:28:47,853 DEBUG [RS:0;c707e9ace675:38827 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,38827,1732663727570 2024-11-26T23:28:47,853 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,38827,1732663727570' 2024-11-26T23:28:47,853 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:28:47,853 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:28:47,854 DEBUG [RS:0;c707e9ace675:38827 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:28:47,854 INFO [RS:0;c707e9ace675:38827 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:28:47,854 INFO [RS:0;c707e9ace675:38827 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:28:47,941 WARN [c707e9ace675:42975 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T23:28:47,956 INFO [RS:0;c707e9ace675:38827 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C38827%2C1732663727570, suffix=, logDir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570, archiveDir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/oldWALs, maxLogs=32 2024-11-26T23:28:47,956 INFO [RS:0;c707e9ace675:38827 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C38827%2C1732663727570.1732663727956 2024-11-26T23:28:47,962 INFO [RS:0;c707e9ace675:38827 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570/c707e9ace675%2C38827%2C1732663727570.1732663727956 2024-11-26T23:28:47,968 DEBUG [RS:0;c707e9ace675:38827 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39207:39207),(127.0.0.1/127.0.0.1:34831:34831)] 2024-11-26T23:28:48,191 DEBUG [c707e9ace675:42975 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T23:28:48,192 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c707e9ace675,38827,1732663727570 2024-11-26T23:28:48,193 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,38827,1732663727570, state=OPENING 2024-11-26T23:28:48,194 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T23:28:48,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:48,196 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:28:48,196 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:28:48,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:28:48,196 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:28:48,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,38827,1732663727570}] 2024-11-26T23:28:48,214 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:48,215 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:48,349 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:28:48,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51511, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:28:48,355 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T23:28:48,355 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:28:48,357 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C38827%2C1732663727570.meta, suffix=.meta, logDir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570, archiveDir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/oldWALs, maxLogs=32 2024-11-26T23:28:48,357 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C38827%2C1732663727570.meta.1732663728357.meta 2024-11-26T23:28:48,368 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570/c707e9ace675%2C38827%2C1732663727570.meta.1732663728357.meta 2024-11-26T23:28:48,373 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34831:34831),(127.0.0.1/127.0.0.1:39207:39207)] 2024-11-26T23:28:48,376 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:28:48,377 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T23:28:48,377 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T23:28:48,377 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T23:28:48,377 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T23:28:48,377 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:28:48,377 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T23:28:48,377 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T23:28:48,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:28:48,379 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:28:48,379 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:48,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:28:48,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:28:48,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:28:48,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:48,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:28:48,381 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:28:48,381 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:28:48,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:48,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:28:48,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:28:48,382 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:28:48,382 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:48,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:28:48,383 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:28:48,384 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740 2024-11-26T23:28:48,385 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740 2024-11-26T23:28:48,386 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:28:48,386 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:28:48,386 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:28:48,387 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:28:48,388 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=708693, jitterRate=-0.09885117411613464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:28:48,388 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T23:28:48,389 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732663728377Writing region info on filesystem at 1732663728377Initializing all the Stores at 1732663728378 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663728378Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663728378Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663728378Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663728378Cleaning up temporary data from old regions at 1732663728386 (+8 ms)Running coprocessor post-open hooks at 1732663728388 (+2 ms)Region opened successfully at 1732663728389 (+1 ms) 2024-11-26T23:28:48,389 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732663728349 2024-11-26T23:28:48,392 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T23:28:48,392 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T23:28:48,393 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,38827,1732663727570 2024-11-26T23:28:48,393 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,38827,1732663727570, state=OPEN 2024-11-26T23:28:48,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:28:48,399 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:28:48,399 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:28:48,399 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:28:48,399 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c707e9ace675,38827,1732663727570 2024-11-26T23:28:48,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T23:28:48,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,38827,1732663727570 in 203 msec 2024-11-26T23:28:48,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T23:28:48,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 614 msec 2024-11-26T23:28:48,405 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:28:48,405 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T23:28:48,406 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:28:48,406 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,38827,1732663727570, seqNum=-1] 2024-11-26T23:28:48,406 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:28:48,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60819, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:28:48,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 661 msec 2024-11-26T23:28:48,413 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732663728413, completionTime=-1 2024-11-26T23:28:48,413 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T23:28:48,413 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T23:28:48,415 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T23:28:48,415 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732663788415 2024-11-26T23:28:48,415 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732663848415 2024-11-26T23:28:48,415 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-26T23:28:48,415 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42975,1732663727516-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:48,415 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42975,1732663727516-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:48,415 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42975,1732663727516-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:48,416 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c707e9ace675:42975, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:48,416 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:48,416 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:48,417 DEBUG [master/c707e9ace675:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T23:28:48,419 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.816sec 2024-11-26T23:28:48,419 INFO [master/c707e9ace675:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T23:28:48,419 INFO [master/c707e9ace675:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T23:28:48,419 INFO [master/c707e9ace675:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T23:28:48,419 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T23:28:48,419 INFO [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T23:28:48,419 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42975,1732663727516-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:28:48,419 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42975,1732663727516-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T23:28:48,422 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T23:28:48,422 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T23:28:48,422 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,42975,1732663727516-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:28:48,487 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@762da951, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:28:48,487 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,42975,-1 for getting cluster id 2024-11-26T23:28:48,487 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:28:48,488 DEBUG [HMaster-EventLoopGroup-14-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5aee37b8-58ee-4cb8-8e4d-f078267a7328' 2024-11-26T23:28:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:28:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5aee37b8-58ee-4cb8-8e4d-f078267a7328" 2024-11-26T23:28:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fa9ca6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:28:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,42975,-1] 2024-11-26T23:28:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:28:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:28:48,490 INFO [HMaster-EventLoopGroup-14-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:28:48,491 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e48481, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:28:48,491 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:28:48,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,38827,1732663727570, seqNum=-1] 2024-11-26T23:28:48,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:28:48,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-15-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58808, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:28:48,494 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c707e9ace675,42975,1732663727516 2024-11-26T23:28:48,495 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:28:48,497 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-26T23:28:48,497 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-26T23:28:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is c707e9ace675,42975,1732663727516 2024-11-26T23:28:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d63bf95 2024-11-26T23:28:48,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-26T23:28:48,499 INFO [HMaster-EventLoopGroup-14-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33008, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-26T23:28:48,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42975 {}] util.TableDescriptorChecker(321): MAX_FILESIZE for table descriptor or "hbase.hregion.max.filesize" (786432) is too small, which might cause over splitting into unmanageable number of regions. 2024-11-26T23:28:48,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42975 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (8192) is too small, which might cause very frequent flushing. 2024-11-26T23:28:48,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42975 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestLogRolling-testLogRolling', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:28:48,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42975 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling 2024-11-26T23:28:48,502 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_PRE_OPERATION 2024-11-26T23:28:48,502 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:48,502 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42975 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestLogRolling-testLogRolling" procId is: 4 2024-11-26T23:28:48,503 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-26T23:28:48,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:28:48,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741835_1011 (size=381) 2024-11-26T23:28:48,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741835_1011 (size=381) 2024-11-26T23:28:48,512 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1c9019e82d17d62edf4591ae3dacb350, NAME => 'TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestLogRolling-testLogRolling', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1 2024-11-26T23:28:48,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741836_1012 (size=64) 2024-11-26T23:28:48,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741836_1012 (size=64) 2024-11-26T23:28:48,521 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:28:48,521 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1722): Closing 1c9019e82d17d62edf4591ae3dacb350, disabling compactions & flushes 2024-11-26T23:28:48,521 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:28:48,521 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:28:48,521 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. after waiting 0 ms 2024-11-26T23:28:48,521 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:28:48,521 INFO [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:28:48,521 DEBUG [RegionOpenAndInit-TestLogRolling-testLogRolling-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1c9019e82d17d62edf4591ae3dacb350: Waiting for close lock at 1732663728521Disabling compacts and flushes for region at 1732663728521Disabling writes for close at 1732663728521Writing region close event to WAL at 1732663728521Closed at 1732663728521 2024-11-26T23:28:48,522 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ADD_TO_META 2024-11-26T23:28:48,523 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732663728522"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732663728522"}]},"ts":"1732663728522"} 2024-11-26T23:28:48,525 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-26T23:28:48,527 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-26T23:28:48,527 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663728527"}]},"ts":"1732663728527"} 2024-11-26T23:28:48,529 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLING in hbase:meta 2024-11-26T23:28:48,529 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1c9019e82d17d62edf4591ae3dacb350, ASSIGN}] 2024-11-26T23:28:48,530 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1c9019e82d17d62edf4591ae3dacb350, ASSIGN 2024-11-26T23:28:48,531 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1c9019e82d17d62edf4591ae3dacb350, ASSIGN; state=OFFLINE, location=c707e9ace675,38827,1732663727570; forceNewPlan=false, retain=false 2024-11-26T23:28:48,682 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1c9019e82d17d62edf4591ae3dacb350, regionState=OPENING, regionLocation=c707e9ace675,38827,1732663727570 2024-11-26T23:28:48,684 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1c9019e82d17d62edf4591ae3dacb350, ASSIGN because future has completed 2024-11-26T23:28:48,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c9019e82d17d62edf4591ae3dacb350, server=c707e9ace675,38827,1732663727570}] 2024-11-26T23:28:48,841 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:28:48,841 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 1c9019e82d17d62edf4591ae3dacb350, NAME => 'TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:28:48,842 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:48,842 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:28:48,842 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:48,842 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:48,843 INFO [StoreOpener-1c9019e82d17d62edf4591ae3dacb350-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:48,844 INFO [StoreOpener-1c9019e82d17d62edf4591ae3dacb350-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1c9019e82d17d62edf4591ae3dacb350 columnFamilyName info 2024-11-26T23:28:48,844 DEBUG [StoreOpener-1c9019e82d17d62edf4591ae3dacb350-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:28:48,845 INFO [StoreOpener-1c9019e82d17d62edf4591ae3dacb350-1 {}] regionserver.HStore(327): Store=1c9019e82d17d62edf4591ae3dacb350/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:28:48,845 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:48,845 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:48,846 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:48,846 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:48,846 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:48,847 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:48,850 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:28:48,850 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 1c9019e82d17d62edf4591ae3dacb350; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=849620, jitterRate=0.08034852147102356}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:28:48,850 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:48,851 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 1c9019e82d17d62edf4591ae3dacb350: Running coprocessor pre-open hook at 1732663728842Writing region info on filesystem at 1732663728842Initializing all the Stores at 1732663728843 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663728843Cleaning up temporary data from old regions at 1732663728846 (+3 ms)Running coprocessor post-open hooks at 1732663728850 (+4 ms)Region opened successfully at 1732663728851 (+1 ms) 2024-11-26T23:28:48,852 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350., pid=6, masterSystemTime=1732663728837 2024-11-26T23:28:48,854 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:28:48,854 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:28:48,855 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=1c9019e82d17d62edf4591ae3dacb350, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,38827,1732663727570 2024-11-26T23:28:48,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1c9019e82d17d62edf4591ae3dacb350, server=c707e9ace675,38827,1732663727570 because future has completed 2024-11-26T23:28:48,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-26T23:28:48,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 1c9019e82d17d62edf4591ae3dacb350, server=c707e9ace675,38827,1732663727570 in 173 msec 2024-11-26T23:28:48,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-26T23:28:48,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1c9019e82d17d62edf4591ae3dacb350, ASSIGN in 332 msec 2024-11-26T23:28:48,865 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-26T23:28:48,865 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestLogRolling-testLogRolling","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732663728865"}]},"ts":"1732663728865"} 2024-11-26T23:28:48,867 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestLogRolling-testLogRolling, state=ENABLED in hbase:meta 2024-11-26T23:28:48,868 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestLogRolling-testLogRolling execute state=CREATE_TABLE_POST_OPERATION 2024-11-26T23:28:48,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestLogRolling-testLogRolling in 368 msec 2024-11-26T23:28:49,215 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:49,215 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:50,216 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:50,216 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:51,216 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:51,216 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:51,310 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,310 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,310 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,310 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,310 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,310 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,311 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,311 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,337 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,338 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,338 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,338 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,338 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,339 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,343 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,344 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,344 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,347 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,853 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-26T23:28:51,854 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,854 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,854 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,855 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,855 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,855 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,855 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,856 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,875 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,875 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,875 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,875 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,875 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,876 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,879 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,879 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,879 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:51,881 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:28:52,217 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:52,217 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:53,218 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:53,218 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:53,818 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T23:28:53,818 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestLogRolling-testLogRolling' 2024-11-26T23:28:54,218 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:54,218 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:55,219 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:55,219 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:55,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-26T23:28:55,734 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling Metrics about Tables on a single HBase RegionServer 2024-11-26T23:28:55,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testCompactionRecordDoesntBlockRolling 2024-11-26T23:28:56,220 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:56,220 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:57,220 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:57,220 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:58,221 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:58,221 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:58,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42975 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-26T23:28:58,527 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestLogRolling-testLogRolling completed 2024-11-26T23:28:58,527 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=TestLogRolling-testLogRolling,, stopping at row=TestLogRolling-testLogRolling ,, for max=2147483647 with caching=100 2024-11-26T23:28:58,530 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table TestLogRolling-testLogRolling 2024-11-26T23:28:58,530 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:28:58,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0001', locateType=CURRENT is [region=TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350., hostname=c707e9ace675,38827,1732663727570, seqNum=2] 2024-11-26T23:28:58,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:58,545 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c9019e82d17d62edf4591ae3dacb350 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-26T23:28:58,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/84a9f1329b1f4f12babe987599f5138c is 1080, key is row0001/info:/1732663738534/Put/seqid=0 2024-11-26T23:28:58,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741837_1013 (size=12509) 2024-11-26T23:28:58,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741837_1013 (size=12509) 2024-11-26T23:28:58,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/84a9f1329b1f4f12babe987599f5138c 2024-11-26T23:28:58,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/84a9f1329b1f4f12babe987599f5138c as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/84a9f1329b1f4f12babe987599f5138c 2024-11-26T23:28:58,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/84a9f1329b1f4f12babe987599f5138c, entries=7, sequenceid=11, filesize=12.2 K 2024-11-26T23:28:58,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=21.02 KB/21520 for 1c9019e82d17d62edf4591ae3dacb350 in 36ms, sequenceid=11, compaction requested=false 2024-11-26T23:28:58,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c9019e82d17d62edf4591ae3dacb350: 2024-11-26T23:28:58,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:28:58,581 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c9019e82d17d62edf4591ae3dacb350 1/1 column families, dataSize=22.07 KB heapSize=23.88 KB 2024-11-26T23:28:58,586 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/18a35c7ec3be469f8568a782f1166054 is 1080, key is row0008/info:/1732663738546/Put/seqid=0 2024-11-26T23:28:58,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741838_1014 (size=27607) 2024-11-26T23:28:58,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741838_1014 (size=27607) 2024-11-26T23:28:58,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.07 KB at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/18a35c7ec3be469f8568a782f1166054 2024-11-26T23:28:58,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/18a35c7ec3be469f8568a782f1166054 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/18a35c7ec3be469f8568a782f1166054 2024-11-26T23:28:58,601 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/18a35c7ec3be469f8568a782f1166054, entries=21, sequenceid=35, filesize=27.0 K 2024-11-26T23:28:58,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~22.07 KB/22596, heapSize ~23.86 KB/24432, currentSize=4.20 KB/4304 for 1c9019e82d17d62edf4591ae3dacb350 in 21ms, sequenceid=35, compaction requested=false 2024-11-26T23:28:58,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c9019e82d17d62edf4591ae3dacb350: 2024-11-26T23:28:58,603 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=39.2 K, sizeToCheck=16.0 K 2024-11-26T23:28:58,603 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:28:58,603 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/18a35c7ec3be469f8568a782f1166054 because midkey is the same as first or last row 2024-11-26T23:28:59,221 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:28:59,221 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:00,222 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:00,222 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:00,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:29:00,595 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c9019e82d17d62edf4591ae3dacb350 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-26T23:29:00,599 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/1df7b2ce916245eea31ee68b73fafddb is 1080, key is row0029/info:/1732663738582/Put/seqid=0 2024-11-26T23:29:00,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741839_1015 (size=12509) 2024-11-26T23:29:00,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741839_1015 (size=12509) 2024-11-26T23:29:00,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=45 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/1df7b2ce916245eea31ee68b73fafddb 2024-11-26T23:29:00,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/1df7b2ce916245eea31ee68b73fafddb as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1df7b2ce916245eea31ee68b73fafddb 2024-11-26T23:29:00,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1df7b2ce916245eea31ee68b73fafddb, entries=7, sequenceid=45, filesize=12.2 K 2024-11-26T23:29:00,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=14.71 KB/15064 for 1c9019e82d17d62edf4591ae3dacb350 in 24ms, sequenceid=45, compaction requested=true 2024-11-26T23:29:00,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c9019e82d17d62edf4591ae3dacb350: 2024-11-26T23:29:00,619 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=51.4 K, sizeToCheck=16.0 K 2024-11-26T23:29:00,619 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:29:00,619 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/18a35c7ec3be469f8568a782f1166054 because midkey is the same as first or last row 2024-11-26T23:29:00,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c9019e82d17d62edf4591ae3dacb350:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:29:00,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:00,619 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:29:00,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:29:00,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c9019e82d17d62edf4591ae3dacb350 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-26T23:29:00,621 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 52625 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:29:00,621 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1541): 1c9019e82d17d62edf4591ae3dacb350/info is initiating minor compaction (all files) 2024-11-26T23:29:00,621 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c9019e82d17d62edf4591ae3dacb350/info in TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:29:00,621 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/84a9f1329b1f4f12babe987599f5138c, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/18a35c7ec3be469f8568a782f1166054, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1df7b2ce916245eea31ee68b73fafddb] into tmpdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp, totalSize=51.4 K 2024-11-26T23:29:00,622 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84a9f1329b1f4f12babe987599f5138c, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1732663738534 2024-11-26T23:29:00,622 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18a35c7ec3be469f8568a782f1166054, keycount=21, bloomtype=ROW, size=27.0 K, encoding=NONE, compression=NONE, seqNum=35, earliestPutTs=1732663738546 2024-11-26T23:29:00,623 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1df7b2ce916245eea31ee68b73fafddb, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732663738582 2024-11-26T23:29:00,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/8556ca7bd48a43279cf906998e9d65df is 1080, key is row0036/info:/1732663740596/Put/seqid=0 2024-11-26T23:29:00,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741840_1016 (size=21141) 2024-11-26T23:29:00,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741840_1016 (size=21141) 2024-11-26T23:29:00,636 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/8556ca7bd48a43279cf906998e9d65df 2024-11-26T23:29:00,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/8556ca7bd48a43279cf906998e9d65df as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/8556ca7bd48a43279cf906998e9d65df 2024-11-26T23:29:00,649 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c9019e82d17d62edf4591ae3dacb350#info#compaction#61 average throughput is 17.96 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:29:00,649 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/ac473c56e96a471f8834753cdf5e7b1c is 1080, key is row0001/info:/1732663738534/Put/seqid=0 2024-11-26T23:29:00,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/8556ca7bd48a43279cf906998e9d65df, entries=15, sequenceid=63, filesize=20.6 K 2024-11-26T23:29:00,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=13.66 KB/13988 for 1c9019e82d17d62edf4591ae3dacb350 in 31ms, sequenceid=63, compaction requested=false 2024-11-26T23:29:00,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c9019e82d17d62edf4591ae3dacb350: 2024-11-26T23:29:00,652 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.0 K, sizeToCheck=16.0 K 2024-11-26T23:29:00,652 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:29:00,652 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/18a35c7ec3be469f8568a782f1166054 because midkey is the same as first or last row 2024-11-26T23:29:00,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:29:00,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1c9019e82d17d62edf4591ae3dacb350 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-26T23:29:00,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/9fd7d4eca58048478f3e8c8ee9cc6710 is 1080, key is row0051/info:/1732663740621/Put/seqid=0 2024-11-26T23:29:00,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741841_1017 (size=42824) 2024-11-26T23:29:00,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741841_1017 (size=42824) 2024-11-26T23:29:00,666 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/ac473c56e96a471f8834753cdf5e7b1c as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/ac473c56e96a471f8834753cdf5e7b1c 2024-11-26T23:29:00,671 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c9019e82d17d62edf4591ae3dacb350/info of 1c9019e82d17d62edf4591ae3dacb350 into ac473c56e96a471f8834753cdf5e7b1c(size=41.8 K), total size for store is 62.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c9019e82d17d62edf4591ae3dacb350: 2024-11-26T23:29:00,672 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350., storeName=1c9019e82d17d62edf4591ae3dacb350/info, priority=13, startTime=1732663740619; duration=0sec 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=62.5 K, sizeToCheck=16.0 K 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/ac473c56e96a471f8834753cdf5e7b1c because midkey is the same as first or last row 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=62.5 K, sizeToCheck=16.0 K 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/ac473c56e96a471f8834753cdf5e7b1c because midkey is the same as first or last row 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=62.5 K, sizeToCheck=16.0 K 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/ac473c56e96a471f8834753cdf5e7b1c because midkey is the same as first or last row 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:00,672 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c9019e82d17d62edf4591ae3dacb350:info 2024-11-26T23:29:00,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741842_1018 (size=20064) 2024-11-26T23:29:00,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741842_1018 (size=20064) 2024-11-26T23:29:00,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/9fd7d4eca58048478f3e8c8ee9cc6710 2024-11-26T23:29:00,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/9fd7d4eca58048478f3e8c8ee9cc6710 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/9fd7d4eca58048478f3e8c8ee9cc6710 2024-11-26T23:29:00,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/9fd7d4eca58048478f3e8c8ee9cc6710, entries=14, sequenceid=80, filesize=19.6 K 2024-11-26T23:29:00,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=0 B/0 for 1c9019e82d17d62edf4591ae3dacb350 in 34ms, sequenceid=80, compaction requested=true 2024-11-26T23:29:00,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1c9019e82d17d62edf4591ae3dacb350: 2024-11-26T23:29:00,688 DEBUG [MemStoreFlusher.0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=82.1 K, sizeToCheck=16.0 K 2024-11-26T23:29:00,688 DEBUG [MemStoreFlusher.0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:29:00,688 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreUtils(137): cannot split hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/ac473c56e96a471f8834753cdf5e7b1c because midkey is the same as first or last row 2024-11-26T23:29:00,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1c9019e82d17d62edf4591ae3dacb350:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:29:00,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:00,688 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:29:00,689 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84029 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:29:00,689 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1541): 1c9019e82d17d62edf4591ae3dacb350/info is initiating minor compaction (all files) 2024-11-26T23:29:00,689 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1c9019e82d17d62edf4591ae3dacb350/info in TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:29:00,689 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/ac473c56e96a471f8834753cdf5e7b1c, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/8556ca7bd48a43279cf906998e9d65df, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/9fd7d4eca58048478f3e8c8ee9cc6710] into tmpdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp, totalSize=82.1 K 2024-11-26T23:29:00,690 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac473c56e96a471f8834753cdf5e7b1c, keycount=35, bloomtype=ROW, size=41.8 K, encoding=NONE, compression=NONE, seqNum=45, earliestPutTs=1732663738534 2024-11-26T23:29:00,690 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8556ca7bd48a43279cf906998e9d65df, keycount=15, bloomtype=ROW, size=20.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1732663740596 2024-11-26T23:29:00,691 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fd7d4eca58048478f3e8c8ee9cc6710, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732663740621 2024-11-26T23:29:00,701 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1c9019e82d17d62edf4591ae3dacb350#info#compaction#63 average throughput is 32.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:29:00,702 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/1832310cc23a46e097a056722a396fba is 1080, key is row0001/info:/1732663738534/Put/seqid=0 2024-11-26T23:29:00,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741843_1019 (size=74301) 2024-11-26T23:29:00,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741843_1019 (size=74301) 2024-11-26T23:29:00,711 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/.tmp/info/1832310cc23a46e097a056722a396fba as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1832310cc23a46e097a056722a396fba 2024-11-26T23:29:00,718 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1c9019e82d17d62edf4591ae3dacb350/info of 1c9019e82d17d62edf4591ae3dacb350 into 1832310cc23a46e097a056722a396fba(size=72.6 K), total size for store is 72.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:29:00,718 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1c9019e82d17d62edf4591ae3dacb350: 2024-11-26T23:29:00,718 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350., storeName=1c9019e82d17d62edf4591ae3dacb350/info, priority=13, startTime=1732663740688; duration=0sec 2024-11-26T23:29:00,718 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K 2024-11-26T23:29:00,718 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:29:00,718 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K 2024-11-26T23:29:00,718 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:29:00,718 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.ConstantSizeRegionSplitPolicy(101): Should split because region size is big enough sumSize=72.6 K, sizeToCheck=16.0 K 2024-11-26T23:29:00,718 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.IncreasingToUpperBoundRegionSplitPolicy(85): regionsWithCommonTable=1 2024-11-26T23:29:00,719 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit(239): Splitting TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350., compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:00,719 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:00,719 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1c9019e82d17d62edf4591ae3dacb350:info 2024-11-26T23:29:00,720 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42975 {}] assignment.AssignmentManager(1363): Split request from c707e9ace675,38827,1732663727570, parent={ENCODED => 1c9019e82d17d62edf4591ae3dacb350, NAME => 'TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.', STARTKEY => '', ENDKEY => ''}, splitKey=row0062 2024-11-26T23:29:00,726 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42975 {}] assignment.SplitTableRegionProcedure(223): Splittable=true state=OPEN, location=c707e9ace675,38827,1732663727570 2024-11-26T23:29:00,730 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42975 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1c9019e82d17d62edf4591ae3dacb350, daughterA=64dc78747b3eab2249ad2e3b448230c2, daughterB=4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:00,731 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1c9019e82d17d62edf4591ae3dacb350, daughterA=64dc78747b3eab2249ad2e3b448230c2, daughterB=4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:00,731 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1c9019e82d17d62edf4591ae3dacb350, daughterA=64dc78747b3eab2249ad2e3b448230c2, daughterB=4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:00,731 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:SPLIT_TABLE_REGION_PREPARE, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1c9019e82d17d62edf4591ae3dacb350, daughterA=64dc78747b3eab2249ad2e3b448230c2, daughterB=4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:00,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1c9019e82d17d62edf4591ae3dacb350, UNASSIGN}] 2024-11-26T23:29:00,739 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1c9019e82d17d62edf4591ae3dacb350, UNASSIGN 2024-11-26T23:29:00,741 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1c9019e82d17d62edf4591ae3dacb350, regionState=CLOSING, regionLocation=c707e9ace675,38827,1732663727570 2024-11-26T23:29:00,743 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1c9019e82d17d62edf4591ae3dacb350, UNASSIGN because future has completed 2024-11-26T23:29:00,743 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: true: evictOnSplit: true: evictOnClose: false 2024-11-26T23:29:00,744 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c9019e82d17d62edf4591ae3dacb350, server=c707e9ace675,38827,1732663727570}] 2024-11-26T23:29:00,901 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(122): Close 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:29:00,901 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(136): Unassign region: split region: true: evictCache: true 2024-11-26T23:29:00,902 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1722): Closing 1c9019e82d17d62edf4591ae3dacb350, disabling compactions & flushes 2024-11-26T23:29:00,902 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:29:00,902 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:29:00,902 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. after waiting 0 ms 2024-11-26T23:29:00,902 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:29:00,903 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/84a9f1329b1f4f12babe987599f5138c, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/18a35c7ec3be469f8568a782f1166054, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/ac473c56e96a471f8834753cdf5e7b1c, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1df7b2ce916245eea31ee68b73fafddb, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/8556ca7bd48a43279cf906998e9d65df, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/9fd7d4eca58048478f3e8c8ee9cc6710] to archive 2024-11-26T23:29:00,905 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:29:00,906 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/84a9f1329b1f4f12babe987599f5138c to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/84a9f1329b1f4f12babe987599f5138c 2024-11-26T23:29:00,908 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/18a35c7ec3be469f8568a782f1166054 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/18a35c7ec3be469f8568a782f1166054 2024-11-26T23:29:00,909 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/ac473c56e96a471f8834753cdf5e7b1c to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/ac473c56e96a471f8834753cdf5e7b1c 2024-11-26T23:29:00,910 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1df7b2ce916245eea31ee68b73fafddb to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1df7b2ce916245eea31ee68b73fafddb 2024-11-26T23:29:00,911 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/8556ca7bd48a43279cf906998e9d65df to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/8556ca7bd48a43279cf906998e9d65df 2024-11-26T23:29:00,912 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/9fd7d4eca58048478f3e8c8ee9cc6710 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/9fd7d4eca58048478f3e8c8ee9cc6710 2024-11-26T23:29:00,918 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=1 2024-11-26T23:29:00,919 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. 2024-11-26T23:29:00,919 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] regionserver.HRegion(1676): Region close journal for 1c9019e82d17d62edf4591ae3dacb350: Waiting for close lock at 1732663740902Running coprocessor pre-close hooks at 1732663740902Disabling compacts and flushes for region at 1732663740902Disabling writes for close at 1732663740902Writing region close event to WAL at 1732663740915 (+13 ms)Running coprocessor post-close hooks at 1732663740919 (+4 ms)Closed at 1732663740919 2024-11-26T23:29:00,921 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION, pid=9}] handler.UnassignRegionHandler(157): Closed 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:29:00,922 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=1c9019e82d17d62edf4591ae3dacb350, regionState=CLOSED 2024-11-26T23:29:00,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=8, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1c9019e82d17d62edf4591ae3dacb350, server=c707e9ace675,38827,1732663727570 because future has completed 2024-11-26T23:29:00,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=8 2024-11-26T23:29:00,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=8, state=SUCCESS, hasLock=false; CloseRegionProcedure 1c9019e82d17d62edf4591ae3dacb350, server=c707e9ace675,38827,1732663727570 in 181 msec 2024-11-26T23:29:00,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-26T23:29:00,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=1c9019e82d17d62edf4591ae3dacb350, UNASSIGN in 189 msec 2024-11-26T23:29:00,937 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:00,940 INFO [PEWorker-4 {}] assignment.SplitTableRegionProcedure(728): pid=7 splitting 1 storefiles, region=1c9019e82d17d62edf4591ae3dacb350, threads=1 2024-11-26T23:29:00,942 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(823): pid=7 splitting started for store file: hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1832310cc23a46e097a056722a396fba for region: 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:29:00,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741844_1020 (size=27) 2024-11-26T23:29:00,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741844_1020 (size=27) 2024-11-26T23:29:00,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741845_1021 (size=27) 2024-11-26T23:29:00,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741845_1021 (size=27) 2024-11-26T23:29:00,971 DEBUG [StoreFileSplitter-pool-0 {}] assignment.SplitTableRegionProcedure(834): pid=7 splitting complete for store file: hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1832310cc23a46e097a056722a396fba for region: 1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:29:00,973 DEBUG [PEWorker-4 {}] assignment.SplitTableRegionProcedure(802): pid=7 split storefiles for region 1c9019e82d17d62edf4591ae3dacb350 Daughter A: [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350] storefiles, Daughter B: [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350] storefiles. 2024-11-26T23:29:00,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741846_1022 (size=71) 2024-11-26T23:29:00,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741846_1022 (size=71) 2024-11-26T23:29:00,985 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:00,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741847_1023 (size=71) 2024-11-26T23:29:00,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741847_1023 (size=71) 2024-11-26T23:29:00,998 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:01,005 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=-1 2024-11-26T23:29:01,007 DEBUG [PEWorker-4 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/recovered.edits/85.seqid, newMaxSeqId=85, maxSeqId=-1 2024-11-26T23:29:01,010 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350.","families":{"info":[{"qualifier":"regioninfo","vlen":63,"tag":[],"timestamp":"1732663741009"},{"qualifier":"splitA","vlen":70,"tag":[],"timestamp":"1732663741009"},{"qualifier":"splitB","vlen":70,"tag":[],"timestamp":"1732663741009"}]},"ts":"1732663741009"} 2024-11-26T23:29:01,010 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732663741009"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732663741009"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732663741009"}]},"ts":"1732663741009"} 2024-11-26T23:29:01,010 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Put {"totalColumns":3,"row":"TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.","families":{"info":[{"qualifier":"regioninfo","vlen":70,"tag":[],"timestamp":"1732663741009"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732663741009"},{"qualifier":"seqnumDuringOpen","vlen":8,"tag":[],"timestamp":"1732663741009"}]},"ts":"1732663741009"} 2024-11-26T23:29:01,029 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=64dc78747b3eab2249ad2e3b448230c2, ASSIGN}, {pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4d69861b88f1f5e423ea06c608a6e142, ASSIGN}] 2024-11-26T23:29:01,031 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=64dc78747b3eab2249ad2e3b448230c2, ASSIGN 2024-11-26T23:29:01,031 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4d69861b88f1f5e423ea06c608a6e142, ASSIGN 2024-11-26T23:29:01,032 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=64dc78747b3eab2249ad2e3b448230c2, ASSIGN; state=SPLITTING_NEW, location=c707e9ace675,38827,1732663727570; forceNewPlan=false, retain=false 2024-11-26T23:29:01,032 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4d69861b88f1f5e423ea06c608a6e142, ASSIGN; state=SPLITTING_NEW, location=c707e9ace675,38827,1732663727570; forceNewPlan=false, retain=false 2024-11-26T23:29:01,183 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4d69861b88f1f5e423ea06c608a6e142, regionState=OPENING, regionLocation=c707e9ace675,38827,1732663727570 2024-11-26T23:29:01,183 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=64dc78747b3eab2249ad2e3b448230c2, regionState=OPENING, regionLocation=c707e9ace675,38827,1732663727570 2024-11-26T23:29:01,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=64dc78747b3eab2249ad2e3b448230c2, ASSIGN because future has completed 2024-11-26T23:29:01,186 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64dc78747b3eab2249ad2e3b448230c2, server=c707e9ace675,38827,1732663727570}] 2024-11-26T23:29:01,186 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4d69861b88f1f5e423ea06c608a6e142, ASSIGN because future has completed 2024-11-26T23:29:01,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d69861b88f1f5e423ea06c608a6e142, server=c707e9ace675,38827,1732663727570}] 2024-11-26T23:29:01,223 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:01,223 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:01,237 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-26T23:29:01,238 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,238 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,238 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,239 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,239 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,239 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,239 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,240 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,259 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,259 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,259 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,259 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,260 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,260 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,263 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,263 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,263 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,265 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:01,342 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2. 2024-11-26T23:29:01,342 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 64dc78747b3eab2249ad2e3b448230c2, NAME => 'TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2.', STARTKEY => '', ENDKEY => 'row0062'} 2024-11-26T23:29:01,342 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:01,342 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:29:01,342 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:01,342 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:01,343 INFO [StoreOpener-64dc78747b3eab2249ad2e3b448230c2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:01,344 INFO [StoreOpener-64dc78747b3eab2249ad2e3b448230c2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64dc78747b3eab2249ad2e3b448230c2 columnFamilyName info 2024-11-26T23:29:01,344 DEBUG [StoreOpener-64dc78747b3eab2249ad2e3b448230c2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:01,354 DEBUG [StoreOpener-64dc78747b3eab2249ad2e3b448230c2-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350->hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1832310cc23a46e097a056722a396fba-bottom 2024-11-26T23:29:01,355 INFO [StoreOpener-64dc78747b3eab2249ad2e3b448230c2-1 {}] regionserver.HStore(327): Store=64dc78747b3eab2249ad2e3b448230c2/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:29:01,355 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:01,356 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:01,357 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:01,357 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:01,357 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:01,359 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:01,360 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 64dc78747b3eab2249ad2e3b448230c2; next sequenceid=86; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=728964, jitterRate=-0.07307478785514832}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:29:01,360 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:01,360 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 64dc78747b3eab2249ad2e3b448230c2: Running coprocessor pre-open hook at 1732663741342Writing region info on filesystem at 1732663741342Initializing all the Stores at 1732663741343 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663741343Cleaning up temporary data from old regions at 1732663741357 (+14 ms)Running coprocessor post-open hooks at 1732663741360 (+3 ms)Region opened successfully at 1732663741360 2024-11-26T23:29:01,361 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2., pid=12, masterSystemTime=1732663741338 2024-11-26T23:29:01,361 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(403): Add compact mark for store 64dc78747b3eab2249ad2e3b448230c2:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:29:01,361 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:01,361 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-26T23:29:01,362 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2. 2024-11-26T23:29:01,362 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1541): 64dc78747b3eab2249ad2e3b448230c2/info is initiating minor compaction (all files) 2024-11-26T23:29:01,362 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 64dc78747b3eab2249ad2e3b448230c2/info in TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2. 2024-11-26T23:29:01,362 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350->hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1832310cc23a46e097a056722a396fba-bottom] into tmpdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/.tmp, totalSize=72.6 K 2024-11-26T23:29:01,363 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350, keycount=32, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1732663738534 2024-11-26T23:29:01,363 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2. 2024-11-26T23:29:01,363 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2. 2024-11-26T23:29:01,364 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(132): Open TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:01,364 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d69861b88f1f5e423ea06c608a6e142, NAME => 'TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.', STARTKEY => 'row0062', ENDKEY => ''} 2024-11-26T23:29:01,364 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestLogRolling-testLogRolling 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:01,364 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(898): Instantiated TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:29:01,364 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7794): checking encryption for 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:01,364 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(7797): checking classloading for 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:01,364 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=64dc78747b3eab2249ad2e3b448230c2, regionState=OPEN, openSeqNum=86, regionLocation=c707e9ace675,38827,1732663727570 2024-11-26T23:29:01,366 INFO [StoreOpener-4d69861b88f1f5e423ea06c608a6e142-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:01,367 INFO [StoreOpener-4d69861b88f1f5e423ea06c608a6e142-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d69861b88f1f5e423ea06c608a6e142 columnFamilyName info 2024-11-26T23:29:01,367 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 1588230740 2024-11-26T23:29:01,367 DEBUG [StoreOpener-4d69861b88f1f5e423ea06c608a6e142-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:01,367 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-26T23:29:01,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.11 KB heapSize=8.96 KB 2024-11-26T23:29:01,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 64dc78747b3eab2249ad2e3b448230c2, server=c707e9ace675,38827,1732663727570 because future has completed 2024-11-26T23:29:01,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-26T23:29:01,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 64dc78747b3eab2249ad2e3b448230c2, server=c707e9ace675,38827,1732663727570 in 183 msec 2024-11-26T23:29:01,373 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=64dc78747b3eab2249ad2e3b448230c2, ASSIGN in 342 msec 2024-11-26T23:29:01,377 DEBUG [StoreOpener-4d69861b88f1f5e423ea06c608a6e142-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350->hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1832310cc23a46e097a056722a396fba-top 2024-11-26T23:29:01,378 INFO [StoreOpener-4d69861b88f1f5e423ea06c608a6e142-1 {}] regionserver.HStore(327): Store=4d69861b88f1f5e423ea06c608a6e142/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:29:01,378 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1038): replaying wal for 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:01,378 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:01,379 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:01,380 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1048): stopping wal replay for 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:01,380 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1060): Cleaning up temporary data for 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:01,382 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1093): writing seq id for 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:01,383 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1114): Opened 4d69861b88f1f5e423ea06c608a6e142; next sequenceid=86; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=784872, jitterRate=-0.001983657479286194}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-26T23:29:01,383 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:01,383 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegion(1006): Region open journal for 4d69861b88f1f5e423ea06c608a6e142: Running coprocessor pre-open hook at 1732663741364Writing region info on filesystem at 1732663741364Initializing all the Stores at 1732663741365 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663741365Cleaning up temporary data from old regions at 1732663741380 (+15 ms)Running coprocessor post-open hooks at 1732663741383 (+3 ms)Region opened successfully at 1732663741383 2024-11-26T23:29:01,383 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 64dc78747b3eab2249ad2e3b448230c2#info#compaction#64 average throughput is 31.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:29:01,384 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/.tmp/info/a652e2b9f7454108ace435237b282152 is 1080, key is row0001/info:/1732663738534/Put/seqid=0 2024-11-26T23:29:01,386 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2236): Post open deploy tasks for TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., pid=13, masterSystemTime=1732663741338 2024-11-26T23:29:01,386 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(403): Add compact mark for store 4d69861b88f1f5e423ea06c608a6e142:info, priority=-2147483648, current under compaction store size is 2 2024-11-26T23:29:01,386 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: Opening Region; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:01,386 DEBUG [RS:0;c707e9ace675:38827-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-26T23:29:01,387 INFO [RS:0;c707e9ace675:38827-longCompactions-0 {}] regionserver.HStore(1527): Keeping/Overriding Compaction request priority to -2147482648 for CF info since it belongs to recently split daughter region TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:01,387 DEBUG [RS:0;c707e9ace675:38827-longCompactions-0 {}] regionserver.HStore(1541): 4d69861b88f1f5e423ea06c608a6e142/info is initiating minor compaction (all files) 2024-11-26T23:29:01,387 INFO [RS:0;c707e9ace675:38827-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d69861b88f1f5e423ea06c608a6e142/info in TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:01,387 INFO [RS:0;c707e9ace675:38827-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350->hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1832310cc23a46e097a056722a396fba-top] into tmpdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp, totalSize=72.6 K 2024-11-26T23:29:01,388 DEBUG [RS:0;c707e9ace675:38827-longCompactions-0 {}] compactions.Compactor(225): Compacting 1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350, keycount=32, bloomtype=ROW, size=72.6 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732663738534 2024-11-26T23:29:01,389 DEBUG [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] regionserver.HRegionServer(2266): Finished post open deploy task for TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:01,389 INFO [RS_OPEN_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_REGION, pid=13}] handler.AssignRegionHandler(153): Opened TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:01,390 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=4d69861b88f1f5e423ea06c608a6e142, regionState=OPEN, openSeqNum=86, regionLocation=c707e9ace675,38827,1732663727570 2024-11-26T23:29:01,392 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-15-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4d69861b88f1f5e423ea06c608a6e142, server=c707e9ace675,38827,1732663727570 because future has completed 2024-11-26T23:29:01,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741848_1024 (size=70862) 2024-11-26T23:29:01,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741848_1024 (size=70862) 2024-11-26T23:29:01,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/info/db0fd94bfadb470e9d67c27ddd2ecb0b is 193, key is TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142./info:regioninfo/1732663741183/Put/seqid=0 2024-11-26T23:29:01,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=13, resume processing ppid=11 2024-11-26T23:29:01,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 4d69861b88f1f5e423ea06c608a6e142, server=c707e9ace675,38827,1732663727570 in 208 msec 2024-11-26T23:29:01,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=7 2024-11-26T23:29:01,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestLogRolling-testLogRolling, region=4d69861b88f1f5e423ea06c608a6e142, ASSIGN in 369 msec 2024-11-26T23:29:01,403 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; SplitTableRegionProcedure table=TestLogRolling-testLogRolling, parent=1c9019e82d17d62edf4591ae3dacb350, daughterA=64dc78747b3eab2249ad2e3b448230c2, daughterB=4d69861b88f1f5e423ea06c608a6e142 in 675 msec 2024-11-26T23:29:01,406 INFO [RS:0;c707e9ace675:38827-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d69861b88f1f5e423ea06c608a6e142#info#compaction#66 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:29:01,406 DEBUG [RS:0;c707e9ace675:38827-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/ee60b603faac43cdb0b9a0b9d831928c is 1080, key is row0062/info:/1732663740647/Put/seqid=0 2024-11-26T23:29:01,411 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/.tmp/info/a652e2b9f7454108ace435237b282152 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/info/a652e2b9f7454108ace435237b282152 2024-11-26T23:29:01,418 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 64dc78747b3eab2249ad2e3b448230c2/info of 64dc78747b3eab2249ad2e3b448230c2 into a652e2b9f7454108ace435237b282152(size=69.2 K), total size for store is 69.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:29:01,418 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 64dc78747b3eab2249ad2e3b448230c2: 2024-11-26T23:29:01,418 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2., storeName=64dc78747b3eab2249ad2e3b448230c2/info, priority=15, startTime=1732663741361; duration=0sec 2024-11-26T23:29:01,418 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:01,418 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 64dc78747b3eab2249ad2e3b448230c2:info 2024-11-26T23:29:01,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741849_1025 (size=9847) 2024-11-26T23:29:01,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741849_1025 (size=9847) 2024-11-26T23:29:01,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.92 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/info/db0fd94bfadb470e9d67c27ddd2ecb0b 2024-11-26T23:29:01,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741850_1026 (size=8260) 2024-11-26T23:29:01,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741850_1026 (size=8260) 2024-11-26T23:29:01,442 DEBUG [RS:0;c707e9ace675:38827-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/ee60b603faac43cdb0b9a0b9d831928c as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/ee60b603faac43cdb0b9a0b9d831928c 2024-11-26T23:29:01,449 INFO [RS:0;c707e9ace675:38827-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 1 (all) file(s) in 4d69861b88f1f5e423ea06c608a6e142/info of 4d69861b88f1f5e423ea06c608a6e142 into ee60b603faac43cdb0b9a0b9d831928c(size=8.1 K), total size for store is 8.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:29:01,449 DEBUG [RS:0;c707e9ace675:38827-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:01,449 INFO [RS:0;c707e9ace675:38827-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., storeName=4d69861b88f1f5e423ea06c608a6e142/info, priority=15, startTime=1732663741386; duration=0sec 2024-11-26T23:29:01,449 DEBUG [RS:0;c707e9ace675:38827-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:01,449 DEBUG [RS:0;c707e9ace675:38827-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d69861b88f1f5e423ea06c608a6e142:info 2024-11-26T23:29:01,452 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/ns/e5c75de24e56425c93a30654bae8d0a8 is 43, key is default/ns:d/1732663728408/Put/seqid=0 2024-11-26T23:29:01,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741851_1027 (size=5153) 2024-11-26T23:29:01,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741851_1027 (size=5153) 2024-11-26T23:29:01,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/ns/e5c75de24e56425c93a30654bae8d0a8 2024-11-26T23:29:01,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/table/d84277fa2c56413c9f2dd7b78eabde05 is 65, key is TestLogRolling-testLogRolling/table:state/1732663728865/Put/seqid=0 2024-11-26T23:29:01,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741852_1028 (size=5340) 2024-11-26T23:29:01,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741852_1028 (size=5340) 2024-11-26T23:29:01,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=122 B at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/table/d84277fa2c56413c9f2dd7b78eabde05 2024-11-26T23:29:01,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/info/db0fd94bfadb470e9d67c27ddd2ecb0b as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/info/db0fd94bfadb470e9d67c27ddd2ecb0b 2024-11-26T23:29:01,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/info/db0fd94bfadb470e9d67c27ddd2ecb0b, entries=30, sequenceid=17, filesize=9.6 K 2024-11-26T23:29:01,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/ns/e5c75de24e56425c93a30654bae8d0a8 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/ns/e5c75de24e56425c93a30654bae8d0a8 2024-11-26T23:29:01,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/ns/e5c75de24e56425c93a30654bae8d0a8, entries=2, sequenceid=17, filesize=5.0 K 2024-11-26T23:29:01,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/table/d84277fa2c56413c9f2dd7b78eabde05 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/table/d84277fa2c56413c9f2dd7b78eabde05 2024-11-26T23:29:01,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/table/d84277fa2c56413c9f2dd7b78eabde05, entries=2, sequenceid=17, filesize=5.2 K 2024-11-26T23:29:01,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~5.11 KB/5234, heapSize ~8.66 KB/8872, currentSize=705 B/705 for 1588230740 in 141ms, sequenceid=17, compaction requested=false 2024-11-26T23:29:01,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-26T23:29:02,223 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:02,223 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:02,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:58808 deadline: 1732663752655, exception=org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. is not online on c707e9ace675,38827,1732663727570 2024-11-26T23:29:02,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350., hostname=c707e9ace675,38827,1732663727570, seqNum=2 , the old value is region=TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350., hostname=c707e9ace675,38827,1732663727570, seqNum=2, error=org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. is not online on c707e9ace675,38827,1732663727570 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:29:02,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350., hostname=c707e9ace675,38827,1732663727570, seqNum=2 is org.apache.hadoop.hbase.NotServingRegionException: org.apache.hadoop.hbase.NotServingRegionException: TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350. is not online on c707e9ace675,38827,1732663727570 at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegionByEncodedName(HRegionServer.java:3186) at org.apache.hadoop.hbase.regionserver.HRegionServer.getRegion(HRegionServer.java:3164) at org.apache.hadoop.hbase.regionserver.RSRpcServices.getRegion(RSRpcServices.java:1413) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2943) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:29:02,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=TestLogRolling-testLogRolling,,1732663728500.1c9019e82d17d62edf4591ae3dacb350., hostname=c707e9ace675,38827,1732663727570, seqNum=2 from cache 2024-11-26T23:29:03,224 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:03,224 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:04,225 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:04,225 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:05,225 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:05,225 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:06,226 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:06,226 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:06,421 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-datanode.properties,hadoop-metrics2.properties 2024-11-26T23:29:06,422 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,422 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,422 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,422 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,422 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,422 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,423 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,423 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,443 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,443 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,443 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,444 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,444 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,444 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,447 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,447 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,448 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:06,450 WARN [HBase-Metrics2-1 {}] impl.FsDatasetImpl(779): Exception thrown while metric collection. Exception : Cannot invoke "java.util.Map.values()" because "this.executors" is null 2024-11-26T23:29:07,227 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:07,227 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:08,227 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:08,227 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:09,228 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:09,228 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:10,228 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:10,228 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:11,229 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:11,229 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:12,230 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:12,230 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:12,704 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestLogRolling-testLogRolling', row='row0065', locateType=CURRENT is [region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., hostname=c707e9ace675,38827,1732663727570, seqNum=86] 2024-11-26T23:29:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:12,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-26T23:29:12,725 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/3231464c2988490ab30059cf4f3726e7 is 1080, key is row0065/info:/1732663752705/Put/seqid=0 2024-11-26T23:29:12,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741853_1029 (size=12509) 2024-11-26T23:29:12,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741853_1029 (size=12509) 2024-11-26T23:29:12,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/3231464c2988490ab30059cf4f3726e7 2024-11-26T23:29:12,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/3231464c2988490ab30059cf4f3726e7 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/3231464c2988490ab30059cf4f3726e7 2024-11-26T23:29:12,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/3231464c2988490ab30059cf4f3726e7, entries=7, sequenceid=96, filesize=12.2 K 2024-11-26T23:29:12,750 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=17.86 KB/18292 for 4d69861b88f1f5e423ea06c608a6e142 in 33ms, sequenceid=96, compaction requested=false 2024-11-26T23:29:12,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:12,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=18.91 KB heapSize=20.50 KB 2024-11-26T23:29:12,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/e6ed47072d454e60a74f8fcab9e8f284 is 1080, key is row0072/info:/1732663752718/Put/seqid=0 2024-11-26T23:29:12,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741854_1030 (size=24376) 2024-11-26T23:29:12,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741854_1030 (size=24376) 2024-11-26T23:29:12,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=18.91 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/e6ed47072d454e60a74f8fcab9e8f284 2024-11-26T23:29:12,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/e6ed47072d454e60a74f8fcab9e8f284 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e6ed47072d454e60a74f8fcab9e8f284 2024-11-26T23:29:12,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e6ed47072d454e60a74f8fcab9e8f284, entries=18, sequenceid=117, filesize=23.8 K 2024-11-26T23:29:12,774 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~18.91 KB/19368, heapSize ~20.48 KB/20976, currentSize=7.36 KB/7532 for 4d69861b88f1f5e423ea06c608a6e142 in 22ms, sequenceid=117, compaction requested=true 2024-11-26T23:29:12,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:12,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d69861b88f1f5e423ea06c608a6e142:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:29:12,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:12,774 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:29:12,775 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45145 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:29:12,775 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1541): 4d69861b88f1f5e423ea06c608a6e142/info is initiating minor compaction (all files) 2024-11-26T23:29:12,775 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d69861b88f1f5e423ea06c608a6e142/info in TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:12,775 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/ee60b603faac43cdb0b9a0b9d831928c, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/3231464c2988490ab30059cf4f3726e7, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e6ed47072d454e60a74f8fcab9e8f284] into tmpdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp, totalSize=44.1 K 2024-11-26T23:29:12,776 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee60b603faac43cdb0b9a0b9d831928c, keycount=3, bloomtype=ROW, size=8.1 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1732663740647 2024-11-26T23:29:12,776 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3231464c2988490ab30059cf4f3726e7, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1732663752705 2024-11-26T23:29:12,776 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6ed47072d454e60a74f8fcab9e8f284, keycount=18, bloomtype=ROW, size=23.8 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732663752718 2024-11-26T23:29:12,787 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d69861b88f1f5e423ea06c608a6e142#info#compaction#71 average throughput is 28.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:29:12,787 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/37c3efd8f1424b61ae739bb25261843e is 1080, key is row0062/info:/1732663740647/Put/seqid=0 2024-11-26T23:29:12,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741855_1031 (size=35349) 2024-11-26T23:29:12,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741855_1031 (size=35349) 2024-11-26T23:29:12,797 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/37c3efd8f1424b61ae739bb25261843e as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/37c3efd8f1424b61ae739bb25261843e 2024-11-26T23:29:12,802 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d69861b88f1f5e423ea06c608a6e142/info of 4d69861b88f1f5e423ea06c608a6e142 into 37c3efd8f1424b61ae739bb25261843e(size=34.5 K), total size for store is 34.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:29:12,802 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:12,802 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., storeName=4d69861b88f1f5e423ea06c608a6e142/info, priority=13, startTime=1732663752774; duration=0sec 2024-11-26T23:29:12,803 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:12,803 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d69861b88f1f5e423ea06c608a6e142:info 2024-11-26T23:29:13,230 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:13,230 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:14,231 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:14,231 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:14,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:14,767 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=8.41 KB heapSize=9.25 KB 2024-11-26T23:29:14,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/6a5efbaa16fa44aa973d7fc8c3992ab5 is 1080, key is row0090/info:/1732663752753/Put/seqid=0 2024-11-26T23:29:14,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741856_1032 (size=13587) 2024-11-26T23:29:14,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741856_1032 (size=13587) 2024-11-26T23:29:14,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.41 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/6a5efbaa16fa44aa973d7fc8c3992ab5 2024-11-26T23:29:14,783 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/6a5efbaa16fa44aa973d7fc8c3992ab5 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6a5efbaa16fa44aa973d7fc8c3992ab5 2024-11-26T23:29:14,788 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6a5efbaa16fa44aa973d7fc8c3992ab5, entries=8, sequenceid=129, filesize=13.3 K 2024-11-26T23:29:14,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~8.41 KB/8608, heapSize ~9.23 KB/9456, currentSize=14.71 KB/15064 for 4d69861b88f1f5e423ea06c608a6e142 in 22ms, sequenceid=129, compaction requested=false 2024-11-26T23:29:14,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:14,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:14,790 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-26T23:29:14,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/429e3a11e42243f8beddd3faa61d8a1c is 1080, key is row0098/info:/1732663754768/Put/seqid=0 2024-11-26T23:29:14,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741857_1033 (size=21156) 2024-11-26T23:29:14,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741857_1033 (size=21156) 2024-11-26T23:29:14,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/429e3a11e42243f8beddd3faa61d8a1c 2024-11-26T23:29:14,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/429e3a11e42243f8beddd3faa61d8a1c as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/429e3a11e42243f8beddd3faa61d8a1c 2024-11-26T23:29:14,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/429e3a11e42243f8beddd3faa61d8a1c, entries=15, sequenceid=147, filesize=20.7 K 2024-11-26T23:29:14,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=13.66 KB/13988 for 4d69861b88f1f5e423ea06c608a6e142 in 21ms, sequenceid=147, compaction requested=true 2024-11-26T23:29:14,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:14,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d69861b88f1f5e423ea06c608a6e142:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:29:14,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:14,811 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:29:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:14,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-26T23:29:14,812 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 70092 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:29:14,812 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1541): 4d69861b88f1f5e423ea06c608a6e142/info is initiating minor compaction (all files) 2024-11-26T23:29:14,813 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d69861b88f1f5e423ea06c608a6e142/info in TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:14,813 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/37c3efd8f1424b61ae739bb25261843e, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6a5efbaa16fa44aa973d7fc8c3992ab5, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/429e3a11e42243f8beddd3faa61d8a1c] into tmpdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp, totalSize=68.4 K 2024-11-26T23:29:14,813 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37c3efd8f1424b61ae739bb25261843e, keycount=28, bloomtype=ROW, size=34.5 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732663740647 2024-11-26T23:29:14,813 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a5efbaa16fa44aa973d7fc8c3992ab5, keycount=8, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1732663752753 2024-11-26T23:29:14,814 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 429e3a11e42243f8beddd3faa61d8a1c, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732663754768 2024-11-26T23:29:14,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/563fbb36503a427db78d4cbd36b6a3e2 is 1080, key is row0113/info:/1732663754791/Put/seqid=0 2024-11-26T23:29:14,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741858_1034 (size=20078) 2024-11-26T23:29:14,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741858_1034 (size=20078) 2024-11-26T23:29:14,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/563fbb36503a427db78d4cbd36b6a3e2 2024-11-26T23:29:14,825 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d69861b88f1f5e423ea06c608a6e142#info#compaction#75 average throughput is 52.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:29:14,826 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/2db4524b02e14428816290c208d5cb5e is 1080, key is row0062/info:/1732663740647/Put/seqid=0 2024-11-26T23:29:14,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/563fbb36503a427db78d4cbd36b6a3e2 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/563fbb36503a427db78d4cbd36b6a3e2 2024-11-26T23:29:14,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/563fbb36503a427db78d4cbd36b6a3e2, entries=14, sequenceid=164, filesize=19.6 K 2024-11-26T23:29:14,834 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=2.10 KB/2152 for 4d69861b88f1f5e423ea06c608a6e142 in 23ms, sequenceid=164, compaction requested=false 2024-11-26T23:29:14,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:14,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741859_1035 (size=60246) 2024-11-26T23:29:14,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741859_1035 (size=60246) 2024-11-26T23:29:14,841 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/2db4524b02e14428816290c208d5cb5e as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/2db4524b02e14428816290c208d5cb5e 2024-11-26T23:29:14,846 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d69861b88f1f5e423ea06c608a6e142/info of 4d69861b88f1f5e423ea06c608a6e142 into 2db4524b02e14428816290c208d5cb5e(size=58.8 K), total size for store is 78.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:29:14,846 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:14,846 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., storeName=4d69861b88f1f5e423ea06c608a6e142/info, priority=13, startTime=1732663754811; duration=0sec 2024-11-26T23:29:14,846 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:14,846 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d69861b88f1f5e423ea06c608a6e142:info 2024-11-26T23:29:15,232 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:15,232 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:16,232 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:16,232 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:16,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:16,824 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-26T23:29:16,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/7f75723b56564157a0bf834dbb267bc0 is 1080, key is row0127/info:/1732663754812/Put/seqid=0 2024-11-26T23:29:16,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741860_1036 (size=12516) 2024-11-26T23:29:16,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741860_1036 (size=12516) 2024-11-26T23:29:16,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/7f75723b56564157a0bf834dbb267bc0 2024-11-26T23:29:16,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/7f75723b56564157a0bf834dbb267bc0 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f75723b56564157a0bf834dbb267bc0 2024-11-26T23:29:16,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f75723b56564157a0bf834dbb267bc0, entries=7, sequenceid=175, filesize=12.2 K 2024-11-26T23:29:16,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=12.61 KB/12912 for 4d69861b88f1f5e423ea06c608a6e142 in 22ms, sequenceid=175, compaction requested=true 2024-11-26T23:29:16,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:16,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d69861b88f1f5e423ea06c608a6e142:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:29:16,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:16,847 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:29:16,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:16,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=13.66 KB heapSize=14.88 KB 2024-11-26T23:29:16,848 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92840 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:29:16,848 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1541): 4d69861b88f1f5e423ea06c608a6e142/info is initiating minor compaction (all files) 2024-11-26T23:29:16,848 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d69861b88f1f5e423ea06c608a6e142/info in TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:16,848 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/2db4524b02e14428816290c208d5cb5e, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/563fbb36503a427db78d4cbd36b6a3e2, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f75723b56564157a0bf834dbb267bc0] into tmpdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp, totalSize=90.7 K 2024-11-26T23:29:16,848 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2db4524b02e14428816290c208d5cb5e, keycount=51, bloomtype=ROW, size=58.8 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1732663740647 2024-11-26T23:29:16,849 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 563fbb36503a427db78d4cbd36b6a3e2, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732663754791 2024-11-26T23:29:16,849 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f75723b56564157a0bf834dbb267bc0, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732663754812 2024-11-26T23:29:16,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/7f26d97f543d415784dc4c1f895b3957 is 1080, key is row0134/info:/1732663756825/Put/seqid=0 2024-11-26T23:29:16,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741861_1037 (size=19000) 2024-11-26T23:29:16,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741861_1037 (size=19000) 2024-11-26T23:29:16,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.66 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/7f26d97f543d415784dc4c1f895b3957 2024-11-26T23:29:16,863 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d69861b88f1f5e423ea06c608a6e142#info#compaction#78 average throughput is 36.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:29:16,864 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/affc60d5910d4e719793e93ed57b46e2 is 1080, key is row0062/info:/1732663740647/Put/seqid=0 2024-11-26T23:29:16,864 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/7f26d97f543d415784dc4c1f895b3957 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f26d97f543d415784dc4c1f895b3957 2024-11-26T23:29:16,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f26d97f543d415784dc4c1f895b3957, entries=13, sequenceid=191, filesize=18.6 K 2024-11-26T23:29:16,870 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~13.66 KB/13988, heapSize ~14.86 KB/15216, currentSize=11.56 KB/11836 for 4d69861b88f1f5e423ea06c608a6e142 in 23ms, sequenceid=191, compaction requested=false 2024-11-26T23:29:16,871 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:16,871 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-26T23:29:16,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741862_1038 (size=83123) 2024-11-26T23:29:16,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741862_1038 (size=83123) 2024-11-26T23:29:16,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/5f94f2ebb0c04338ad7ff9c2a528a17e is 1080, key is row0147/info:/1732663756848/Put/seqid=0 2024-11-26T23:29:16,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741863_1039 (size=17906) 2024-11-26T23:29:16,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741863_1039 (size=17906) 2024-11-26T23:29:16,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/5f94f2ebb0c04338ad7ff9c2a528a17e 2024-11-26T23:29:16,881 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/affc60d5910d4e719793e93ed57b46e2 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/affc60d5910d4e719793e93ed57b46e2 2024-11-26T23:29:16,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/5f94f2ebb0c04338ad7ff9c2a528a17e as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/5f94f2ebb0c04338ad7ff9c2a528a17e 2024-11-26T23:29:16,888 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d69861b88f1f5e423ea06c608a6e142/info of 4d69861b88f1f5e423ea06c608a6e142 into affc60d5910d4e719793e93ed57b46e2(size=81.2 K), total size for store is 99.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:29:16,888 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:16,888 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., storeName=4d69861b88f1f5e423ea06c608a6e142/info, priority=13, startTime=1732663756847; duration=0sec 2024-11-26T23:29:16,889 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:16,889 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d69861b88f1f5e423ea06c608a6e142:info 2024-11-26T23:29:16,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/5f94f2ebb0c04338ad7ff9c2a528a17e, entries=12, sequenceid=206, filesize=17.5 K 2024-11-26T23:29:16,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=2.10 KB/2152 for 4d69861b88f1f5e423ea06c608a6e142 in 20ms, sequenceid=206, compaction requested=true 2024-11-26T23:29:16,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:16,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d69861b88f1f5e423ea06c608a6e142:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:29:16,892 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:16,892 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:29:16,893 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120029 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:29:16,893 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1541): 4d69861b88f1f5e423ea06c608a6e142/info is initiating minor compaction (all files) 2024-11-26T23:29:16,893 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d69861b88f1f5e423ea06c608a6e142/info in TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:16,893 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/affc60d5910d4e719793e93ed57b46e2, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f26d97f543d415784dc4c1f895b3957, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/5f94f2ebb0c04338ad7ff9c2a528a17e] into tmpdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp, totalSize=117.2 K 2024-11-26T23:29:16,893 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting affc60d5910d4e719793e93ed57b46e2, keycount=72, bloomtype=ROW, size=81.2 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1732663740647 2024-11-26T23:29:16,894 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f26d97f543d415784dc4c1f895b3957, keycount=13, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732663756825 2024-11-26T23:29:16,894 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f94f2ebb0c04338ad7ff9c2a528a17e, keycount=12, bloomtype=ROW, size=17.5 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732663756848 2024-11-26T23:29:16,903 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d69861b88f1f5e423ea06c608a6e142#info#compaction#80 average throughput is 99.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:29:16,904 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/e5283054fb164876a5a27d680adf4c85 is 1080, key is row0062/info:/1732663740647/Put/seqid=0 2024-11-26T23:29:16,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741864_1040 (size=110183) 2024-11-26T23:29:16,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741864_1040 (size=110183) 2024-11-26T23:29:16,913 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/e5283054fb164876a5a27d680adf4c85 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e5283054fb164876a5a27d680adf4c85 2024-11-26T23:29:16,917 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d69861b88f1f5e423ea06c608a6e142/info of 4d69861b88f1f5e423ea06c608a6e142 into e5283054fb164876a5a27d680adf4c85(size=107.6 K), total size for store is 107.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:29:16,917 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:16,917 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., storeName=4d69861b88f1f5e423ea06c608a6e142/info, priority=13, startTime=1732663756892; duration=0sec 2024-11-26T23:29:16,917 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:16,917 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d69861b88f1f5e423ea06c608a6e142:info 2024-11-26T23:29:17,233 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:17,233 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:17,497 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-26T23:29:18,233 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:18,233 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:18,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-26T23:29:18,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/1ccb4daa27334697a79993ebe4e3567d is 1080, key is row0159/info:/1732663756872/Put/seqid=0 2024-11-26T23:29:18,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741865_1041 (size=12516) 2024-11-26T23:29:18,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741865_1041 (size=12516) 2024-11-26T23:29:18,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=218 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/1ccb4daa27334697a79993ebe4e3567d 2024-11-26T23:29:18,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/1ccb4daa27334697a79993ebe4e3567d as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1ccb4daa27334697a79993ebe4e3567d 2024-11-26T23:29:18,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1ccb4daa27334697a79993ebe4e3567d, entries=7, sequenceid=218, filesize=12.2 K 2024-11-26T23:29:18,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for 4d69861b88f1f5e423ea06c608a6e142 in 22ms, sequenceid=218, compaction requested=false 2024-11-26T23:29:18,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:18,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-26T23:29:18,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/692af380a8d8424a933d8346e579eefe is 1080, key is row0166/info:/1732663758884/Put/seqid=0 2024-11-26T23:29:18,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741866_1042 (size=20078) 2024-11-26T23:29:18,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741866_1042 (size=20078) 2024-11-26T23:29:18,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/692af380a8d8424a933d8346e579eefe 2024-11-26T23:29:18,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/692af380a8d8424a933d8346e579eefe as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/692af380a8d8424a933d8346e579eefe 2024-11-26T23:29:18,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/692af380a8d8424a933d8346e579eefe, entries=14, sequenceid=235, filesize=19.6 K 2024-11-26T23:29:18,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=13.66 KB/13988 for 4d69861b88f1f5e423ea06c608a6e142 in 21ms, sequenceid=235, compaction requested=true 2024-11-26T23:29:18,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:18,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d69861b88f1f5e423ea06c608a6e142:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:29:18,926 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:18,926 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:29:18,927 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 142777 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:29:18,927 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1541): 4d69861b88f1f5e423ea06c608a6e142/info is initiating minor compaction (all files) 2024-11-26T23:29:18,928 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d69861b88f1f5e423ea06c608a6e142/info in TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:18,928 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e5283054fb164876a5a27d680adf4c85, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1ccb4daa27334697a79993ebe4e3567d, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/692af380a8d8424a933d8346e579eefe] into tmpdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp, totalSize=139.4 K 2024-11-26T23:29:18,928 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5283054fb164876a5a27d680adf4c85, keycount=97, bloomtype=ROW, size=107.6 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1732663740647 2024-11-26T23:29:18,928 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ccb4daa27334697a79993ebe4e3567d, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=218, earliestPutTs=1732663756872 2024-11-26T23:29:18,929 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 692af380a8d8424a933d8346e579eefe, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732663758884 2024-11-26T23:29:18,939 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d69861b88f1f5e423ea06c608a6e142#info#compaction#83 average throughput is 60.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:29:18,939 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/50a36b8263364f34955b2d2db493ab8d is 1080, key is row0062/info:/1732663740647/Put/seqid=0 2024-11-26T23:29:18,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741867_1043 (size=133055) 2024-11-26T23:29:18,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741867_1043 (size=133055) 2024-11-26T23:29:18,949 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/50a36b8263364f34955b2d2db493ab8d as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/50a36b8263364f34955b2d2db493ab8d 2024-11-26T23:29:18,954 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d69861b88f1f5e423ea06c608a6e142/info of 4d69861b88f1f5e423ea06c608a6e142 into 50a36b8263364f34955b2d2db493ab8d(size=129.9 K), total size for store is 129.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:29:18,954 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:18,954 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., storeName=4d69861b88f1f5e423ea06c608a6e142/info, priority=13, startTime=1732663758926; duration=0sec 2024-11-26T23:29:18,954 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:18,954 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d69861b88f1f5e423ea06c608a6e142:info 2024-11-26T23:29:19,234 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:19,234 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:20,235 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:20,235 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:20,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:20,927 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-26T23:29:20,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/7d0c7d5f9ab849759370c2446d8950f7 is 1080, key is row0180/info:/1732663758906/Put/seqid=0 2024-11-26T23:29:20,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741868_1044 (size=20078) 2024-11-26T23:29:20,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741868_1044 (size=20078) 2024-11-26T23:29:20,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/7d0c7d5f9ab849759370c2446d8950f7 2024-11-26T23:29:20,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/7d0c7d5f9ab849759370c2446d8950f7 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7d0c7d5f9ab849759370c2446d8950f7 2024-11-26T23:29:20,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7d0c7d5f9ab849759370c2446d8950f7, entries=14, sequenceid=253, filesize=19.6 K 2024-11-26T23:29:20,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=14.71 KB/15064 for 4d69861b88f1f5e423ea06c608a6e142 in 25ms, sequenceid=253, compaction requested=false 2024-11-26T23:29:20,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:20,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-26T23:29:20,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/6eb8f5bd9623432ba547079444b72fa9 is 1080, key is row0194/info:/1732663760928/Put/seqid=0 2024-11-26T23:29:20,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741869_1045 (size=21170) 2024-11-26T23:29:20,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741869_1045 (size=21170) 2024-11-26T23:29:20,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/6eb8f5bd9623432ba547079444b72fa9 2024-11-26T23:29:20,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4d69861b88f1f5e423ea06c608a6e142, server=c707e9ace675,38827,1732663727570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-26T23:29:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 1.2 K connection: 172.17.0.2:58808 deadline: 1732663770978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4d69861b88f1f5e423ea06c608a6e142, server=c707e9ace675,38827,1732663727570 2024-11-26T23:29:20,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., hostname=c707e9ace675,38827,1732663727570, seqNum=86 , the old value is region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., hostname=c707e9ace675,38827,1732663727570, seqNum=86, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4d69861b88f1f5e423ea06c608a6e142, server=c707e9ace675,38827,1732663727570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:29:20,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., hostname=c707e9ace675,38827,1732663727570, seqNum=86 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=32.0 K, regionName=4d69861b88f1f5e423ea06c608a6e142, server=c707e9ace675,38827,1732663727570 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-26T23:29:20,979 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., hostname=c707e9ace675,38827,1732663727570, seqNum=86 because the exception is null or not the one we care about 2024-11-26T23:29:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/6eb8f5bd9623432ba547079444b72fa9 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6eb8f5bd9623432ba547079444b72fa9 2024-11-26T23:29:20,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6eb8f5bd9623432ba547079444b72fa9, entries=15, sequenceid=271, filesize=20.7 K 2024-11-26T23:29:20,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=14.71 KB/15064 for 4d69861b88f1f5e423ea06c608a6e142 in 34ms, sequenceid=271, compaction requested=true 2024-11-26T23:29:20,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:20,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d69861b88f1f5e423ea06c608a6e142:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:29:20,986 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:20,986 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:29:20,987 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 174303 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:29:20,987 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1541): 4d69861b88f1f5e423ea06c608a6e142/info is initiating minor compaction (all files) 2024-11-26T23:29:20,987 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d69861b88f1f5e423ea06c608a6e142/info in TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:20,987 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/50a36b8263364f34955b2d2db493ab8d, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7d0c7d5f9ab849759370c2446d8950f7, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6eb8f5bd9623432ba547079444b72fa9] into tmpdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp, totalSize=170.2 K 2024-11-26T23:29:20,987 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50a36b8263364f34955b2d2db493ab8d, keycount=118, bloomtype=ROW, size=129.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732663740647 2024-11-26T23:29:20,988 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d0c7d5f9ab849759370c2446d8950f7, keycount=14, bloomtype=ROW, size=19.6 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732663758906 2024-11-26T23:29:20,988 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eb8f5bd9623432ba547079444b72fa9, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732663760928 2024-11-26T23:29:20,998 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d69861b88f1f5e423ea06c608a6e142#info#compaction#86 average throughput is 75.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:29:20,999 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/bb7a652056ab4dd3b6e746ecc0f06b11 is 1080, key is row0062/info:/1732663740647/Put/seqid=0 2024-11-26T23:29:21,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741870_1046 (size=164510) 2024-11-26T23:29:21,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741870_1046 (size=164510) 2024-11-26T23:29:21,008 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/bb7a652056ab4dd3b6e746ecc0f06b11 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/bb7a652056ab4dd3b6e746ecc0f06b11 2024-11-26T23:29:21,015 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d69861b88f1f5e423ea06c608a6e142/info of 4d69861b88f1f5e423ea06c608a6e142 into bb7a652056ab4dd3b6e746ecc0f06b11(size=160.7 K), total size for store is 160.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:29:21,015 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:21,015 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., storeName=4d69861b88f1f5e423ea06c608a6e142/info, priority=13, startTime=1732663760986; duration=0sec 2024-11-26T23:29:21,016 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:21,016 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d69861b88f1f5e423ea06c608a6e142:info 2024-11-26T23:29:21,235 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:21,235 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:22,236 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:22,236 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:23,237 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:23,237 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:24,237 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:24,237 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:25,238 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:25,238 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:26,238 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:26,238 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:27,239 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:27,239 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:28,240 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:28,240 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:29,198 INFO [master/c707e9ace675:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-26T23:29:29,198 INFO [master/c707e9ace675:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-26T23:29:29,240 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:29,240 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:30,241 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:30,241 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:31,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:31,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=15.76 KB heapSize=17.13 KB 2024-11-26T23:29:31,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/dc25540d4be2440890e0e7ef1f38b962 is 1080, key is row0209/info:/1732663760953/Put/seqid=0 2024-11-26T23:29:31,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741871_1047 (size=21171) 2024-11-26T23:29:31,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741871_1047 (size=21171) 2024-11-26T23:29:31,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.76 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/dc25540d4be2440890e0e7ef1f38b962 2024-11-26T23:29:31,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/dc25540d4be2440890e0e7ef1f38b962 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/dc25540d4be2440890e0e7ef1f38b962 2024-11-26T23:29:31,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/dc25540d4be2440890e0e7ef1f38b962, entries=15, sequenceid=290, filesize=20.7 K 2024-11-26T23:29:31,039 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~15.76 KB/16140, heapSize ~17.11 KB/17520, currentSize=1.05 KB/1076 for 4d69861b88f1f5e423ea06c608a6e142 in 21ms, sequenceid=290, compaction requested=false 2024-11-26T23:29:31,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:31,241 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:31,241 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:32,242 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:32,242 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:33,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:33,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=7.36 KB heapSize=8.13 KB 2024-11-26T23:29:33,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/e8f4b60689e1407da8378cb1c2bd33d7 is 1080, key is row0224/info:/1732663771019/Put/seqid=0 2024-11-26T23:29:33,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741872_1048 (size=12523) 2024-11-26T23:29:33,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741872_1048 (size=12523) 2024-11-26T23:29:33,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.36 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/e8f4b60689e1407da8378cb1c2bd33d7 2024-11-26T23:29:33,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/e8f4b60689e1407da8378cb1c2bd33d7 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e8f4b60689e1407da8378cb1c2bd33d7 2024-11-26T23:29:33,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e8f4b60689e1407da8378cb1c2bd33d7, entries=7, sequenceid=300, filesize=12.2 K 2024-11-26T23:29:33,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.36 KB/7532, heapSize ~8.11 KB/8304, currentSize=13.66 KB/13988 for 4d69861b88f1f5e423ea06c608a6e142 in 22ms, sequenceid=300, compaction requested=true 2024-11-26T23:29:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4d69861b88f1f5e423ea06c608a6e142:info, priority=-2147483648, current under compaction store size is 1 2024-11-26T23:29:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:33,051 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-26T23:29:33,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38827 {}] regionserver.HRegion(8855): Flush requested on 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:33,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=14.71 KB heapSize=16 KB 2024-11-26T23:29:33,052 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 198204 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-26T23:29:33,052 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1541): 4d69861b88f1f5e423ea06c608a6e142/info is initiating minor compaction (all files) 2024-11-26T23:29:33,052 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4d69861b88f1f5e423ea06c608a6e142/info in TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:33,052 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/bb7a652056ab4dd3b6e746ecc0f06b11, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/dc25540d4be2440890e0e7ef1f38b962, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e8f4b60689e1407da8378cb1c2bd33d7] into tmpdir=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp, totalSize=193.6 K 2024-11-26T23:29:33,053 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb7a652056ab4dd3b6e746ecc0f06b11, keycount=147, bloomtype=ROW, size=160.7 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732663740647 2024-11-26T23:29:33,053 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc25540d4be2440890e0e7ef1f38b962, keycount=15, bloomtype=ROW, size=20.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732663760953 2024-11-26T23:29:33,054 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8f4b60689e1407da8378cb1c2bd33d7, keycount=7, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1732663771019 2024-11-26T23:29:33,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/95f410568fe4469a89312f1ea993228d is 1080, key is row0231/info:/1732663773030/Put/seqid=0 2024-11-26T23:29:33,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741873_1049 (size=20092) 2024-11-26T23:29:33,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741873_1049 (size=20092) 2024-11-26T23:29:33,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.71 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/95f410568fe4469a89312f1ea993228d 2024-11-26T23:29:33,068 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4d69861b88f1f5e423ea06c608a6e142#info#compaction#90 average throughput is 86.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-26T23:29:33,069 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/d90e65ef6d634ee2b3750f5a2299e8e0 is 1080, key is row0062/info:/1732663740647/Put/seqid=0 2024-11-26T23:29:33,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/95f410568fe4469a89312f1ea993228d as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/95f410568fe4469a89312f1ea993228d 2024-11-26T23:29:33,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/95f410568fe4469a89312f1ea993228d, entries=14, sequenceid=317, filesize=19.6 K 2024-11-26T23:29:33,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~14.71 KB/15064, heapSize ~15.98 KB/16368, currentSize=12.61 KB/12912 for 4d69861b88f1f5e423ea06c608a6e142 in 24ms, sequenceid=317, compaction requested=false 2024-11-26T23:29:33,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:33,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741874_1050 (size=188358) 2024-11-26T23:29:33,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741874_1050 (size=188358) 2024-11-26T23:29:33,084 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/d90e65ef6d634ee2b3750f5a2299e8e0 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/d90e65ef6d634ee2b3750f5a2299e8e0 2024-11-26T23:29:33,090 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4d69861b88f1f5e423ea06c608a6e142/info of 4d69861b88f1f5e423ea06c608a6e142 into d90e65ef6d634ee2b3750f5a2299e8e0(size=183.9 K), total size for store is 203.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-26T23:29:33,090 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:33,090 INFO [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., storeName=4d69861b88f1f5e423ea06c608a6e142/info, priority=13, startTime=1732663773051; duration=0sec 2024-11-26T23:29:33,090 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-26T23:29:33,090 DEBUG [RS:0;c707e9ace675:38827-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4d69861b88f1f5e423ea06c608a6e142:info 2024-11-26T23:29:33,243 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:33,243 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:33,377 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(329): Region 1588230740, had cached 0 bytes from a total of 20340 2024-11-26T23:29:34,243 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:34,243 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:35,073 INFO [Time-limited test {}] wal.AbstractTestLogRolling(285): after writing there are 0 log files 2024-11-26T23:29:35,073 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C38827%2C1732663727570.1732663775073 2024-11-26T23:29:35,079 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,079 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,079 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,079 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,080 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,080 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570/c707e9ace675%2C38827%2C1732663727570.1732663727956 with entries=311, filesize=307.82 KB; new WAL /user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570/c707e9ace675%2C38827%2C1732663727570.1732663775073 2024-11-26T23:29:35,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741833_1009 (size=315213) 2024-11-26T23:29:35,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741833_1009 (size=315213) 2024-11-26T23:29:35,085 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:34831:34831),(127.0.0.1/127.0.0.1:39207:39207)] 2024-11-26T23:29:35,089 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4d69861b88f1f5e423ea06c608a6e142 1/1 column families, dataSize=12.61 KB heapSize=13.75 KB 2024-11-26T23:29:35,093 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/27ab878daa964ce9ab2416a55406998d is 1080, key is row0245/info:/1732663773054/Put/seqid=0 2024-11-26T23:29:35,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741876_1052 (size=17918) 2024-11-26T23:29:35,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741876_1052 (size=17918) 2024-11-26T23:29:35,098 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=12.61 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/27ab878daa964ce9ab2416a55406998d 2024-11-26T23:29:35,102 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/.tmp/info/27ab878daa964ce9ab2416a55406998d as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/27ab878daa964ce9ab2416a55406998d 2024-11-26T23:29:35,107 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/27ab878daa964ce9ab2416a55406998d, entries=12, sequenceid=333, filesize=17.5 K 2024-11-26T23:29:35,108 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~12.61 KB/12912, heapSize ~13.73 KB/14064, currentSize=0 B/0 for 4d69861b88f1f5e423ea06c608a6e142 in 18ms, sequenceid=333, compaction requested=true 2024-11-26T23:29:35,108 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4d69861b88f1f5e423ea06c608a6e142: 2024-11-26T23:29:35,108 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=705 B heapSize=2.05 KB 2024-11-26T23:29:35,111 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/info/55d0b132b1b6408fb01041e00ff10e82 is 193, key is TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142./info:regioninfo/1732663741390/Put/seqid=0 2024-11-26T23:29:35,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741877_1053 (size=6223) 2024-11-26T23:29:35,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741877_1053 (size=6223) 2024-11-26T23:29:35,116 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=705 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/info/55d0b132b1b6408fb01041e00ff10e82 2024-11-26T23:29:35,121 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/.tmp/info/55d0b132b1b6408fb01041e00ff10e82 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/info/55d0b132b1b6408fb01041e00ff10e82 2024-11-26T23:29:35,128 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/info/55d0b132b1b6408fb01041e00ff10e82, entries=5, sequenceid=21, filesize=6.1 K 2024-11-26T23:29:35,129 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~705 B/705, heapSize ~1.29 KB/1320, currentSize=0 B/0 for 1588230740 in 21ms, sequenceid=21, compaction requested=false 2024-11-26T23:29:35,129 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1588230740: 2024-11-26T23:29:35,129 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 64dc78747b3eab2249ad2e3b448230c2: 2024-11-26T23:29:35,130 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C38827%2C1732663727570.1732663775130 2024-11-26T23:29:35,135 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,135 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,135 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,135 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,135 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,135 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570/c707e9ace675%2C38827%2C1732663727570.1732663775073 with entries=2, filesize=723 B; new WAL /user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570/c707e9ace675%2C38827%2C1732663727570.1732663775130 2024-11-26T23:29:35,136 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:39207:39207),(127.0.0.1/127.0.0.1:34831:34831)] 2024-11-26T23:29:35,136 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570/c707e9ace675%2C38827%2C1732663727570.1732663775073 is not closed yet, will try archiving it next time 2024-11-26T23:29:35,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741875_1051 (size=731) 2024-11-26T23:29:35,137 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570/c707e9ace675%2C38827%2C1732663727570.1732663727956 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/oldWALs/c707e9ace675%2C38827%2C1732663727570.1732663727956 2024-11-26T23:29:35,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741875_1051 (size=731) 2024-11-26T23:29:35,137 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [5,000] milli-secs(wait.for.ratio=[1]) 2024-11-26T23:29:35,138 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/WALs/c707e9ace675,38827,1732663727570/c707e9ace675%2C38827%2C1732663727570.1732663775073 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/oldWALs/c707e9ace675%2C38827%2C1732663727570.1732663775073 2024-11-26T23:29:35,238 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T23:29:35,238 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:29:35,238 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:29:35,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:29:35,238 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:29:35,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:29:35,238 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T23:29:35,238 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1789817915, stopped=false 2024-11-26T23:29:35,238 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c707e9ace675,42975,1732663727516 2024-11-26T23:29:35,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:29:35,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:29:35,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:35,240 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:35,240 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:29:35,241 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:29:35,241 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:29:35,241 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:29:35,241 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,38827,1732663727570' ***** 2024-11-26T23:29:35,241 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T23:29:35,241 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:29:35,241 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:29:35,241 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:29:35,242 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:29:35,242 INFO [RS:0;c707e9ace675:38827 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T23:29:35,242 INFO [RS:0;c707e9ace675:38827 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T23:29:35,242 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(3091): Received CLOSE for 4d69861b88f1f5e423ea06c608a6e142 2024-11-26T23:29:35,242 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(3091): Received CLOSE for 64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:35,242 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4d69861b88f1f5e423ea06c608a6e142, disabling compactions & flushes 2024-11-26T23:29:35,242 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(959): stopping server c707e9ace675,38827,1732663727570 2024-11-26T23:29:35,242 INFO [RS:0;c707e9ace675:38827 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:29:35,242 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:35,242 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:35,242 INFO [RS:0;c707e9ace675:38827 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c707e9ace675:38827. 2024-11-26T23:29:35,242 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. after waiting 0 ms 2024-11-26T23:29:35,242 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:35,242 DEBUG [RS:0;c707e9ace675:38827 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:29:35,242 DEBUG [RS:0;c707e9ace675:38827 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:29:35,242 INFO [RS:0;c707e9ace675:38827 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:29:35,243 INFO [RS:0;c707e9ace675:38827 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:29:35,243 INFO [RS:0;c707e9ace675:38827 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:29:35,243 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T23:29:35,243 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(1321): Waiting on 3 regions to close 2024-11-26T23:29:35,243 DEBUG [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(1325): Online Regions={4d69861b88f1f5e423ea06c608a6e142=TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142., 1588230740=hbase:meta,,1.1588230740, 64dc78747b3eab2249ad2e3b448230c2=TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2.} 2024-11-26T23:29:35,243 DEBUG [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 4d69861b88f1f5e423ea06c608a6e142, 64dc78747b3eab2249ad2e3b448230c2 2024-11-26T23:29:35,243 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:29:35,243 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:29:35,243 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:29:35,243 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:29:35,243 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:29:35,243 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350->hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1832310cc23a46e097a056722a396fba-top, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/ee60b603faac43cdb0b9a0b9d831928c, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/3231464c2988490ab30059cf4f3726e7, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/37c3efd8f1424b61ae739bb25261843e, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e6ed47072d454e60a74f8fcab9e8f284, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6a5efbaa16fa44aa973d7fc8c3992ab5, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/2db4524b02e14428816290c208d5cb5e, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/429e3a11e42243f8beddd3faa61d8a1c, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/563fbb36503a427db78d4cbd36b6a3e2, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/affc60d5910d4e719793e93ed57b46e2, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f75723b56564157a0bf834dbb267bc0, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f26d97f543d415784dc4c1f895b3957, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e5283054fb164876a5a27d680adf4c85, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/5f94f2ebb0c04338ad7ff9c2a528a17e, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1ccb4daa27334697a79993ebe4e3567d, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/50a36b8263364f34955b2d2db493ab8d, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/692af380a8d8424a933d8346e579eefe, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7d0c7d5f9ab849759370c2446d8950f7, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/bb7a652056ab4dd3b6e746ecc0f06b11, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6eb8f5bd9623432ba547079444b72fa9, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/dc25540d4be2440890e0e7ef1f38b962, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e8f4b60689e1407da8378cb1c2bd33d7] to archive 2024-11-26T23:29:35,244 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:35,244 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:35,244 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:29:35,246 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:29:35,247 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/ee60b603faac43cdb0b9a0b9d831928c to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/ee60b603faac43cdb0b9a0b9d831928c 2024-11-26T23:29:35,248 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-26T23:29:35,248 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:29:35,248 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:29:35,249 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663775243Running coprocessor pre-close hooks at 1732663775243Disabling compacts and flushes for region at 1732663775243Disabling writes for close at 1732663775243Writing region close event to WAL at 1732663775245 (+2 ms)Running coprocessor post-close hooks at 1732663775248 (+3 ms)Closed at 1732663775248 2024-11-26T23:29:35,249 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/3231464c2988490ab30059cf4f3726e7 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/3231464c2988490ab30059cf4f3726e7 2024-11-26T23:29:35,249 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T23:29:35,250 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/37c3efd8f1424b61ae739bb25261843e to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/37c3efd8f1424b61ae739bb25261843e 2024-11-26T23:29:35,251 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e6ed47072d454e60a74f8fcab9e8f284 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e6ed47072d454e60a74f8fcab9e8f284 2024-11-26T23:29:35,252 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6a5efbaa16fa44aa973d7fc8c3992ab5 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6a5efbaa16fa44aa973d7fc8c3992ab5 2024-11-26T23:29:35,253 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/2db4524b02e14428816290c208d5cb5e to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/2db4524b02e14428816290c208d5cb5e 2024-11-26T23:29:35,254 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/429e3a11e42243f8beddd3faa61d8a1c to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/429e3a11e42243f8beddd3faa61d8a1c 2024-11-26T23:29:35,255 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/563fbb36503a427db78d4cbd36b6a3e2 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/563fbb36503a427db78d4cbd36b6a3e2 2024-11-26T23:29:35,257 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/affc60d5910d4e719793e93ed57b46e2 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/affc60d5910d4e719793e93ed57b46e2 2024-11-26T23:29:35,258 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f75723b56564157a0bf834dbb267bc0 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f75723b56564157a0bf834dbb267bc0 2024-11-26T23:29:35,259 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f26d97f543d415784dc4c1f895b3957 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7f26d97f543d415784dc4c1f895b3957 2024-11-26T23:29:35,260 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e5283054fb164876a5a27d680adf4c85 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e5283054fb164876a5a27d680adf4c85 2024-11-26T23:29:35,261 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/5f94f2ebb0c04338ad7ff9c2a528a17e to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/5f94f2ebb0c04338ad7ff9c2a528a17e 2024-11-26T23:29:35,262 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1ccb4daa27334697a79993ebe4e3567d to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/1ccb4daa27334697a79993ebe4e3567d 2024-11-26T23:29:35,263 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/50a36b8263364f34955b2d2db493ab8d to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/50a36b8263364f34955b2d2db493ab8d 2024-11-26T23:29:35,264 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/692af380a8d8424a933d8346e579eefe to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/692af380a8d8424a933d8346e579eefe 2024-11-26T23:29:35,265 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7d0c7d5f9ab849759370c2446d8950f7 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/7d0c7d5f9ab849759370c2446d8950f7 2024-11-26T23:29:35,266 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/bb7a652056ab4dd3b6e746ecc0f06b11 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/bb7a652056ab4dd3b6e746ecc0f06b11 2024-11-26T23:29:35,268 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6eb8f5bd9623432ba547079444b72fa9 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/6eb8f5bd9623432ba547079444b72fa9 2024-11-26T23:29:35,269 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/dc25540d4be2440890e0e7ef1f38b962 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/dc25540d4be2440890e0e7ef1f38b962 2024-11-26T23:29:35,270 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e8f4b60689e1407da8378cb1c2bd33d7 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/info/e8f4b60689e1407da8378cb1c2bd33d7 2024-11-26T23:29:35,270 DEBUG [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] regionserver.HRegionServer(3532): Failed to report file archival(s) to Master. This will be retried. org.apache.hadoop.hbase.ipc.StoppedRpcClientException: Call to address=c707e9ace675:42975 failed on local exception: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.IPCUtil.wrapException(IPCUtil.java:237) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:395) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:451) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callBlockingMethod(AbstractRpcClient.java:336) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$BlockingRpcChannelImplementation.callBlockingMethod(AbstractRpcClient.java:606) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$BlockingStub.reportFileArchival(RegionServerStatusProtos.java:17350) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportFileArchivalForQuotas(HRegionServer.java:3516) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.reportArchivedFilesForQuota(HStore.java:2412) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.removeCompactedfiles(HStore.java:2347) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.closeWithoutLock(HStore.java:738) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.close(HStore.java:804) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1912) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion$2.call(HRegion.java:1909) ~[classes/:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) ~[?:?] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.StoppedRpcClientException at org.apache.hadoop.hbase.ipc.AbstractRpcClient.getConnection(AbstractRpcClient.java:366) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.callMethod(AbstractRpcClient.java:448) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 16 more 2024-11-26T23:29:35,271 WARN [StoreCloser-TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [ee60b603faac43cdb0b9a0b9d831928c=8260, 3231464c2988490ab30059cf4f3726e7=12509, 37c3efd8f1424b61ae739bb25261843e=35349, e6ed47072d454e60a74f8fcab9e8f284=24376, 6a5efbaa16fa44aa973d7fc8c3992ab5=13587, 2db4524b02e14428816290c208d5cb5e=60246, 429e3a11e42243f8beddd3faa61d8a1c=21156, 563fbb36503a427db78d4cbd36b6a3e2=20078, affc60d5910d4e719793e93ed57b46e2=83123, 7f75723b56564157a0bf834dbb267bc0=12516, 7f26d97f543d415784dc4c1f895b3957=19000, e5283054fb164876a5a27d680adf4c85=110183, 5f94f2ebb0c04338ad7ff9c2a528a17e=17906, 1ccb4daa27334697a79993ebe4e3567d=12516, 50a36b8263364f34955b2d2db493ab8d=133055, 692af380a8d8424a933d8346e579eefe=20078, 7d0c7d5f9ab849759370c2446d8950f7=20078, bb7a652056ab4dd3b6e746ecc0f06b11=164510, 6eb8f5bd9623432ba547079444b72fa9=21170, dc25540d4be2440890e0e7ef1f38b962=21171, e8f4b60689e1407da8378cb1c2bd33d7=12523] 2024-11-26T23:29:35,274 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/4d69861b88f1f5e423ea06c608a6e142/recovered.edits/336.seqid, newMaxSeqId=336, maxSeqId=85 2024-11-26T23:29:35,275 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:35,275 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4d69861b88f1f5e423ea06c608a6e142: Waiting for close lock at 1732663775242Running coprocessor pre-close hooks at 1732663775242Disabling compacts and flushes for region at 1732663775242Disabling writes for close at 1732663775242Writing region close event to WAL at 1732663775271 (+29 ms)Running coprocessor post-close hooks at 1732663775275 (+4 ms)Closed at 1732663775275 2024-11-26T23:29:35,275 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,row0062,1732663740726.4d69861b88f1f5e423ea06c608a6e142. 2024-11-26T23:29:35,275 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 64dc78747b3eab2249ad2e3b448230c2, disabling compactions & flushes 2024-11-26T23:29:35,275 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2. 2024-11-26T23:29:35,275 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2. 2024-11-26T23:29:35,275 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2. after waiting 0 ms 2024-11-26T23:29:35,275 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2. 2024-11-26T23:29:35,275 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350->hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/1c9019e82d17d62edf4591ae3dacb350/info/1832310cc23a46e097a056722a396fba-bottom] to archive 2024-11-26T23:29:35,276 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-26T23:29:35,278 DEBUG [StoreCloser-TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350 to hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/archive/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/info/1832310cc23a46e097a056722a396fba.1c9019e82d17d62edf4591ae3dacb350 2024-11-26T23:29:35,278 WARN [StoreCloser-TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2.-1 {}] regionserver.HStore(2414): Failed to report archival of files: [] 2024-11-26T23:29:35,281 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/data/default/TestLogRolling-testLogRolling/64dc78747b3eab2249ad2e3b448230c2/recovered.edits/90.seqid, newMaxSeqId=90, maxSeqId=85 2024-11-26T23:29:35,281 INFO [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2. 2024-11-26T23:29:35,281 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 64dc78747b3eab2249ad2e3b448230c2: Waiting for close lock at 1732663775275Running coprocessor pre-close hooks at 1732663775275Disabling compacts and flushes for region at 1732663775275Disabling writes for close at 1732663775275Writing region close event to WAL at 1732663775278 (+3 ms)Running coprocessor post-close hooks at 1732663775281 (+3 ms)Closed at 1732663775281 2024-11-26T23:29:35,282 DEBUG [RS_CLOSE_REGION-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed TestLogRolling-testLogRolling,,1732663740726.64dc78747b3eab2249ad2e3b448230c2. 2024-11-26T23:29:35,443 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,38827,1732663727570; all regions closed. 2024-11-26T23:29:35,444 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,444 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,444 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,444 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,444 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741834_1010 (size=8107) 2024-11-26T23:29:35,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741834_1010 (size=8107) 2024-11-26T23:29:35,449 DEBUG [RS:0;c707e9ace675:38827 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/oldWALs 2024-11-26T23:29:35,449 INFO [RS:0;c707e9ace675:38827 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C38827%2C1732663727570.meta:.meta(num 1732663728357) 2024-11-26T23:29:35,449 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,449 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,449 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,449 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,449 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741878_1054 (size=778) 2024-11-26T23:29:35,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741878_1054 (size=778) 2024-11-26T23:29:35,453 DEBUG [RS:0;c707e9ace675:38827 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/oldWALs 2024-11-26T23:29:35,453 INFO [RS:0;c707e9ace675:38827 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C38827%2C1732663727570:(num 1732663775130) 2024-11-26T23:29:35,453 DEBUG [RS:0;c707e9ace675:38827 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:29:35,453 INFO [RS:0;c707e9ace675:38827 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:29:35,453 INFO [RS:0;c707e9ace675:38827 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:29:35,453 INFO [RS:0;c707e9ace675:38827 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T23:29:35,453 INFO [RS:0;c707e9ace675:38827 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:29:35,453 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:29:35,454 INFO [RS:0;c707e9ace675:38827 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38827 2024-11-26T23:29:35,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:29:35,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,38827,1732663727570 2024-11-26T23:29:35,457 INFO [RS:0;c707e9ace675:38827 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:29:35,458 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,38827,1732663727570] 2024-11-26T23:29:35,460 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,38827,1732663727570 already deleted, retry=false 2024-11-26T23:29:35,460 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c707e9ace675,38827,1732663727570 expired; onlineServers=0 2024-11-26T23:29:35,460 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c707e9ace675,42975,1732663727516' ***** 2024-11-26T23:29:35,460 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T23:29:35,460 INFO [M:0;c707e9ace675:42975 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:29:35,460 INFO [M:0;c707e9ace675:42975 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:29:35,460 DEBUG [M:0;c707e9ace675:42975 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T23:29:35,460 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T23:29:35,460 DEBUG [M:0;c707e9ace675:42975 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T23:29:35,460 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663727757 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663727757,5,FailOnTimeoutGroup] 2024-11-26T23:29:35,460 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663727757 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663727757,5,FailOnTimeoutGroup] 2024-11-26T23:29:35,460 INFO [M:0;c707e9ace675:42975 {}] hbase.ChoreService(370): Chore service for: master/c707e9ace675:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T23:29:35,460 INFO [M:0;c707e9ace675:42975 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:29:35,460 DEBUG [M:0;c707e9ace675:42975 {}] master.HMaster(1795): Stopping service threads 2024-11-26T23:29:35,460 INFO [M:0;c707e9ace675:42975 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T23:29:35,460 INFO [M:0;c707e9ace675:42975 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:29:35,461 INFO [M:0;c707e9ace675:42975 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T23:29:35,461 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T23:29:35,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T23:29:35,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:35,462 DEBUG [M:0;c707e9ace675:42975 {}] zookeeper.ZKUtil(347): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T23:29:35,462 WARN [M:0;c707e9ace675:42975 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T23:29:35,462 INFO [M:0;c707e9ace675:42975 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/.lastflushedseqids 2024-11-26T23:29:35,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741879_1055 (size=228) 2024-11-26T23:29:35,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741879_1055 (size=228) 2024-11-26T23:29:35,468 INFO [M:0;c707e9ace675:42975 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T23:29:35,468 INFO [M:0;c707e9ace675:42975 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T23:29:35,468 DEBUG [M:0;c707e9ace675:42975 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:29:35,468 INFO [M:0;c707e9ace675:42975 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:35,468 DEBUG [M:0;c707e9ace675:42975 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:35,468 DEBUG [M:0;c707e9ace675:42975 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:29:35,468 DEBUG [M:0;c707e9ace675:42975 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:35,468 INFO [M:0;c707e9ace675:42975 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=51.43 KB heapSize=63.36 KB 2024-11-26T23:29:35,484 DEBUG [M:0;c707e9ace675:42975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1575c40988b448c8fd7e1ac58fa3058 is 82, key is hbase:meta,,1/info:regioninfo/1732663728393/Put/seqid=0 2024-11-26T23:29:35,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741880_1056 (size=5672) 2024-11-26T23:29:35,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741880_1056 (size=5672) 2024-11-26T23:29:35,489 INFO [M:0;c707e9ace675:42975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1575c40988b448c8fd7e1ac58fa3058 2024-11-26T23:29:35,508 DEBUG [M:0;c707e9ace675:42975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7cca0bc0ddf34601b6c024d4c78af18e is 751, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732663728869/Put/seqid=0 2024-11-26T23:29:35,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741881_1057 (size=7091) 2024-11-26T23:29:35,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741881_1057 (size=7091) 2024-11-26T23:29:35,514 INFO [M:0;c707e9ace675:42975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=50.83 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7cca0bc0ddf34601b6c024d4c78af18e 2024-11-26T23:29:35,518 INFO [M:0;c707e9ace675:42975 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7cca0bc0ddf34601b6c024d4c78af18e 2024-11-26T23:29:35,533 DEBUG [M:0;c707e9ace675:42975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3288a08a6221483885e403dfaecf2baf is 69, key is c707e9ace675,38827,1732663727570/rs:state/1732663727808/Put/seqid=0 2024-11-26T23:29:35,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741882_1058 (size=5156) 2024-11-26T23:29:35,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741882_1058 (size=5156) 2024-11-26T23:29:35,538 INFO [M:0;c707e9ace675:42975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3288a08a6221483885e403dfaecf2baf 2024-11-26T23:29:35,556 DEBUG [M:0;c707e9ace675:42975 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/18f56d2fcee44c9aba1fb4658c835fb0 is 52, key is load_balancer_on/state:d/1732663728496/Put/seqid=0 2024-11-26T23:29:35,558 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:29:35,558 INFO [RS:0;c707e9ace675:38827 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:29:35,559 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38827-0x101569c401a0001, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:29:35,559 INFO [RS:0;c707e9ace675:38827 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,38827,1732663727570; zookeeper connection closed. 2024-11-26T23:29:35,559 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@22946422 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@22946422 2024-11-26T23:29:35,559 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T23:29:35,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741883_1059 (size=5056) 2024-11-26T23:29:35,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741883_1059 (size=5056) 2024-11-26T23:29:35,562 INFO [M:0;c707e9ace675:42975 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/18f56d2fcee44c9aba1fb4658c835fb0 2024-11-26T23:29:35,566 DEBUG [M:0;c707e9ace675:42975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/b1575c40988b448c8fd7e1ac58fa3058 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1575c40988b448c8fd7e1ac58fa3058 2024-11-26T23:29:35,570 INFO [M:0;c707e9ace675:42975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/b1575c40988b448c8fd7e1ac58fa3058, entries=8, sequenceid=125, filesize=5.5 K 2024-11-26T23:29:35,571 DEBUG [M:0;c707e9ace675:42975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/7cca0bc0ddf34601b6c024d4c78af18e as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7cca0bc0ddf34601b6c024d4c78af18e 2024-11-26T23:29:35,575 INFO [M:0;c707e9ace675:42975 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 7cca0bc0ddf34601b6c024d4c78af18e 2024-11-26T23:29:35,575 INFO [M:0;c707e9ace675:42975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/7cca0bc0ddf34601b6c024d4c78af18e, entries=13, sequenceid=125, filesize=6.9 K 2024-11-26T23:29:35,576 DEBUG [M:0;c707e9ace675:42975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/3288a08a6221483885e403dfaecf2baf as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3288a08a6221483885e403dfaecf2baf 2024-11-26T23:29:35,584 INFO [M:0;c707e9ace675:42975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/3288a08a6221483885e403dfaecf2baf, entries=1, sequenceid=125, filesize=5.0 K 2024-11-26T23:29:35,584 DEBUG [M:0;c707e9ace675:42975 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/18f56d2fcee44c9aba1fb4658c835fb0 as hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/18f56d2fcee44c9aba1fb4658c835fb0 2024-11-26T23:29:35,588 INFO [M:0;c707e9ace675:42975 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44869/user/jenkins/test-data/ff2cacbb-30a7-6a6f-189d-6b84b1da93a1/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/18f56d2fcee44c9aba1fb4658c835fb0, entries=1, sequenceid=125, filesize=4.9 K 2024-11-26T23:29:35,589 INFO [M:0;c707e9ace675:42975 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=125, compaction requested=false 2024-11-26T23:29:35,591 INFO [M:0;c707e9ace675:42975 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:35,591 DEBUG [M:0;c707e9ace675:42975 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663775468Disabling compacts and flushes for region at 1732663775468Disabling writes for close at 1732663775468Obtaining lock to block concurrent updates at 1732663775468Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732663775468Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=52663, getHeapSize=64816, getOffHeapSize=0, getCellsCount=148 at 1732663775469 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732663775469Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732663775469Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732663775483 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732663775483Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732663775493 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732663775508 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732663775508Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732663775518 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732663775532 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732663775532Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732663775542 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732663775556 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732663775556Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cf4392a: reopening flushed file at 1732663775566 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6d61d30b: reopening flushed file at 1732663775571 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22dbf92: reopening flushed file at 1732663775575 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@48e5b8c5: reopening flushed file at 1732663775584 (+9 ms)Finished flush of dataSize ~51.43 KB/52663, heapSize ~63.30 KB/64816, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 121ms, sequenceid=125, compaction requested=false at 1732663775589 (+5 ms)Writing region close event to WAL at 1732663775591 (+2 ms)Closed at 1732663775591 2024-11-26T23:29:35,591 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,591 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,591 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,591 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,591 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:35,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43015 is added to blk_1073741830_1006 (size=61332) 2024-11-26T23:29:35,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33689 is added to blk_1073741830_1006 (size=61332) 2024-11-26T23:29:35,594 INFO [M:0;c707e9ace675:42975 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T23:29:35,594 INFO [M:0;c707e9ace675:42975 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42975 2024-11-26T23:29:35,594 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:29:35,594 INFO [M:0;c707e9ace675:42975 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:29:35,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:29:35,697 INFO [M:0;c707e9ace675:42975 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:29:35,697 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42975-0x101569c401a0000, quorum=127.0.0.1:54328, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:29:35,700 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@28377467{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:29:35,700 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@41a74ab6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:29:35,700 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:29:35,700 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3612be31{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:29:35,700 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5689196f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/hadoop.log.dir/,STOPPED} 2024-11-26T23:29:35,702 WARN [BP-893640581-172.17.0.2-1732663726836 heartbeating to localhost/127.0.0.1:44869 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:29:35,702 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:29:35,703 WARN [BP-893640581-172.17.0.2-1732663726836 heartbeating to localhost/127.0.0.1:44869 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-893640581-172.17.0.2-1732663726836 (Datanode Uuid e6138815-011f-4ed6-88b6-260564e409ce) service to localhost/127.0.0.1:44869 2024-11-26T23:29:35,703 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:29:35,703 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4/data/data3/current/BP-893640581-172.17.0.2-1732663726836 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:29:35,703 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4/data/data4/current/BP-893640581-172.17.0.2-1732663726836 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:29:35,704 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:29:35,706 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@26881465{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:29:35,706 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5ecf816b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:29:35,706 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:29:35,706 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595aaa8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:29:35,706 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2d8c7847{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/hadoop.log.dir/,STOPPED} 2024-11-26T23:29:35,709 WARN [BP-893640581-172.17.0.2-1732663726836 heartbeating to localhost/127.0.0.1:44869 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:29:35,709 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:29:35,709 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:29:35,709 WARN [BP-893640581-172.17.0.2-1732663726836 heartbeating to localhost/127.0.0.1:44869 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-893640581-172.17.0.2-1732663726836 (Datanode Uuid 38d37e44-0c61-4580-8e9e-599b5e47f98d) service to localhost/127.0.0.1:44869 2024-11-26T23:29:35,709 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4/data/data1/current/BP-893640581-172.17.0.2-1732663726836 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:29:35,710 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/cluster_efa71236-afcf-12fe-0b99-bb524386f2f4/data/data2/current/BP-893640581-172.17.0.2-1732663726836 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:29:35,710 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:29:35,716 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6e1ebd6b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:29:35,717 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1a0fc9ad{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:29:35,717 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:29:35,717 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@188ddc10{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:29:35,717 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@62c26df6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/hadoop.log.dir/,STOPPED} 2024-11-26T23:29:35,727 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T23:29:35,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:29:35,734 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-26T23:29:35,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-26T23:29:35,735 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestLogRolling-testLogRolling 2024-11-26T23:29:35,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T23:29:35,771 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRolling Thread=232 (was 206) Potentially hanging thread: nioEventLoopGroup-39-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44869 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-38-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44869 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-38-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44869 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44869 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-15-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44869 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-39-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-41-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:44869 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:44869 from jenkins.hfs.6 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-38-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-40-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44869 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-39-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44869 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.6@localhost:44869 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-14-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=512 (was 483) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=79 (was 57) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6421 (was 6637) 2024-11-26T23:29:35,780 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=232, OpenFileDescriptor=512, MaxFileDescriptor=1048576, SystemLoadAverage=79, ProcessCount=11, AvailableMemoryMB=6421 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=2, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/hadoop.log.dir so I do NOT create it in target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(401): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6597ab31-b986-2bbb-8b89-594d57dbdd95/hadoop.tmp.dir so I do NOT create it in target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd, deleteOnExit=true 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/test.cache.data in system properties and HBase conf 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/hadoop.tmp.dir in system properties and HBase conf 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/hadoop.log.dir in system properties and HBase conf 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-26T23:29:35,781 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:29:35,781 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/nfs.dump.dir in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/java.io.tmpdir in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-26T23:29:35,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-26T23:29:35,795 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:29:35,824 INFO [regionserver/c707e9ace675:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:29:35,854 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:29:35,857 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:29:35,859 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:29:35,859 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:29:35,859 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-26T23:29:35,861 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:29:35,861 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4d118eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:29:35,861 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5031c9de{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:29:35,976 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3885c0c5{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/java.io.tmpdir/jetty-localhost-44889-hadoop-hdfs-3_4_1-tests_jar-_-any-9043985771330463451/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:29:35,976 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2394ff19{HTTP/1.1, (http/1.1)}{localhost:44889} 2024-11-26T23:29:35,976 INFO [Time-limited test {}] server.Server(415): Started @285321ms 2024-11-26T23:29:35,989 WARN [Time-limited test {}] blockmanagement.DatanodeManager(468): The given interval for marking stale datanode = 30000, which is larger than heartbeat expire interval 20000. 2024-11-26T23:29:36,065 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:29:36,067 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:29:36,068 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:29:36,068 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:29:36,068 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:29:36,069 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40f3733a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:29:36,069 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6162294b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:29:36,183 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2482618b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/java.io.tmpdir/jetty-localhost-41969-hadoop-hdfs-3_4_1-tests_jar-_-any-14021489865082850310/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:29:36,184 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@56836339{HTTP/1.1, (http/1.1)}{localhost:41969} 2024-11-26T23:29:36,184 INFO [Time-limited test {}] server.Server(415): Started @285529ms 2024-11-26T23:29:36,185 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:29:36,217 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-26T23:29:36,219 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-26T23:29:36,220 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-26T23:29:36,220 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-26T23:29:36,220 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-26T23:29:36,220 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@e9b8f9f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/hadoop.log.dir/,AVAILABLE} 2024-11-26T23:29:36,221 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@47b9368{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-26T23:29:36,244 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:36,244 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:36,281 WARN [Thread-2483 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd/data/data2/current/BP-364958726-172.17.0.2-1732663775802/current, will proceed with Du for space computation calculation, 2024-11-26T23:29:36,281 WARN [Thread-2482 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd/data/data1/current/BP-364958726-172.17.0.2-1732663775802/current, will proceed with Du for space computation calculation, 2024-11-26T23:29:36,302 WARN [Thread-2461 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:29:36,305 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x597c42228c52dacc with lease ID 0xcb7e4ad0c46faec1: Processing first storage report for DS-ec4b1062-5d26-4c22-929f-5bf1a170e9c8 from datanode DatanodeRegistration(127.0.0.1:43415, datanodeUuid=bcbd0e77-2b0d-45c8-a337-86b48680f756, infoPort=36725, infoSecurePort=0, ipcPort=35051, storageInfo=lv=-57;cid=testClusterID;nsid=1628253671;c=1732663775802) 2024-11-26T23:29:36,305 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x597c42228c52dacc with lease ID 0xcb7e4ad0c46faec1: from storage DS-ec4b1062-5d26-4c22-929f-5bf1a170e9c8 node DatanodeRegistration(127.0.0.1:43415, datanodeUuid=bcbd0e77-2b0d-45c8-a337-86b48680f756, infoPort=36725, infoSecurePort=0, ipcPort=35051, storageInfo=lv=-57;cid=testClusterID;nsid=1628253671;c=1732663775802), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:29:36,305 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x597c42228c52dacc with lease ID 0xcb7e4ad0c46faec1: Processing first storage report for DS-0f5e7ad3-4884-4c12-813e-8588dd3fd75b from datanode DatanodeRegistration(127.0.0.1:43415, datanodeUuid=bcbd0e77-2b0d-45c8-a337-86b48680f756, infoPort=36725, infoSecurePort=0, ipcPort=35051, storageInfo=lv=-57;cid=testClusterID;nsid=1628253671;c=1732663775802) 2024-11-26T23:29:36,305 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x597c42228c52dacc with lease ID 0xcb7e4ad0c46faec1: from storage DS-0f5e7ad3-4884-4c12-813e-8588dd3fd75b node DatanodeRegistration(127.0.0.1:43415, datanodeUuid=bcbd0e77-2b0d-45c8-a337-86b48680f756, infoPort=36725, infoSecurePort=0, ipcPort=35051, storageInfo=lv=-57;cid=testClusterID;nsid=1628253671;c=1732663775802), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:29:36,340 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@663b7fb1{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/java.io.tmpdir/jetty-localhost-40117-hadoop-hdfs-3_4_1-tests_jar-_-any-590978137772928607/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:29:36,340 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@644054b{HTTP/1.1, (http/1.1)}{localhost:40117} 2024-11-26T23:29:36,340 INFO [Time-limited test {}] server.Server(415): Started @285685ms 2024-11-26T23:29:36,342 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-26T23:29:36,449 WARN [Thread-2509 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd/data/data4/current/BP-364958726-172.17.0.2-1732663775802/current, will proceed with Du for space computation calculation, 2024-11-26T23:29:36,449 WARN [Thread-2508 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd/data/data3/current/BP-364958726-172.17.0.2-1732663775802/current, will proceed with Du for space computation calculation, 2024-11-26T23:29:36,466 WARN [Thread-2497 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-26T23:29:36,468 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7ce39a4d51eedbcc with lease ID 0xcb7e4ad0c46faec2: Processing first storage report for DS-e9980e57-1c1d-40da-8a18-7e860d94ce18 from datanode DatanodeRegistration(127.0.0.1:39645, datanodeUuid=f6b642ef-988d-4f85-b9f3-d1cfe9844422, infoPort=46361, infoSecurePort=0, ipcPort=34355, storageInfo=lv=-57;cid=testClusterID;nsid=1628253671;c=1732663775802) 2024-11-26T23:29:36,468 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7ce39a4d51eedbcc with lease ID 0xcb7e4ad0c46faec2: from storage DS-e9980e57-1c1d-40da-8a18-7e860d94ce18 node DatanodeRegistration(127.0.0.1:39645, datanodeUuid=f6b642ef-988d-4f85-b9f3-d1cfe9844422, infoPort=46361, infoSecurePort=0, ipcPort=34355, storageInfo=lv=-57;cid=testClusterID;nsid=1628253671;c=1732663775802), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:29:36,468 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7ce39a4d51eedbcc with lease ID 0xcb7e4ad0c46faec2: Processing first storage report for DS-bd65ce55-1306-410a-a74f-50d7c01a227d from datanode DatanodeRegistration(127.0.0.1:39645, datanodeUuid=f6b642ef-988d-4f85-b9f3-d1cfe9844422, infoPort=46361, infoSecurePort=0, ipcPort=34355, storageInfo=lv=-57;cid=testClusterID;nsid=1628253671;c=1732663775802) 2024-11-26T23:29:36,468 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7ce39a4d51eedbcc with lease ID 0xcb7e4ad0c46faec2: from storage DS-bd65ce55-1306-410a-a74f-50d7c01a227d node DatanodeRegistration(127.0.0.1:39645, datanodeUuid=f6b642ef-988d-4f85-b9f3-d1cfe9844422, infoPort=46361, infoSecurePort=0, ipcPort=34355, storageInfo=lv=-57;cid=testClusterID;nsid=1628253671;c=1732663775802), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-26T23:29:36,563 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8 2024-11-26T23:29:36,566 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd/zookeeper_0, clientPort=60826, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-26T23:29:36,567 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=60826 2024-11-26T23:29:36,567 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:29:36,568 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:29:36,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:29:36,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741825_1001 (size=7) 2024-11-26T23:29:36,579 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b with version=8 2024-11-26T23:29:36,579 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1141): The hbase.fs.tmp.dir is set to hdfs://localhost:34219/user/jenkins/test-data/8c3840ba-a36c-b27b-2825-15cca517420d/hbase-staging 2024-11-26T23:29:36,581 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:29:36,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:29:36,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:29:36,581 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:29:36,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:29:36,581 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:29:36,581 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-26T23:29:36,581 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:29:36,582 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44051 2024-11-26T23:29:36,583 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:44051 connecting to ZooKeeper ensemble=127.0.0.1:60826 2024-11-26T23:29:36,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:440510x0, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:29:36,592 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:44051-0x101569cffc50000 connected 2024-11-26T23:29:36,614 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:29:36,615 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:29:36,617 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:29:36,618 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b, hbase.cluster.distributed=false 2024-11-26T23:29:36,620 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:29:36,620 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44051 2024-11-26T23:29:36,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44051 2024-11-26T23:29:36,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44051 2024-11-26T23:29:36,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44051 2024-11-26T23:29:36,621 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44051 2024-11-26T23:29:36,636 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c707e9ace675:0 server-side Connection retries=45 2024-11-26T23:29:36,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:29:36,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-26T23:29:36,636 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-26T23:29:36,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-26T23:29:36,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-26T23:29:36,636 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-26T23:29:36,637 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-26T23:29:36,639 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36141 2024-11-26T23:29:36,640 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36141 connecting to ZooKeeper ensemble=127.0.0.1:60826 2024-11-26T23:29:36,640 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:29:36,642 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:29:36,646 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361410x0, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-26T23:29:36,647 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36141-0x101569cffc50001 connected 2024-11-26T23:29:36,647 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:29:36,647 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-26T23:29:36,648 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-26T23:29:36,649 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-26T23:29:36,650 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-26T23:29:36,650 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36141 2024-11-26T23:29:36,652 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36141 2024-11-26T23:29:36,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36141 2024-11-26T23:29:36,659 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36141 2024-11-26T23:29:36,659 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36141 2024-11-26T23:29:36,672 DEBUG [M:0;c707e9ace675:44051 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c707e9ace675:44051 2024-11-26T23:29:36,672 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c707e9ace675,44051,1732663776581 2024-11-26T23:29:36,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:29:36,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:29:36,674 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c707e9ace675,44051,1732663776581 2024-11-26T23:29:36,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-26T23:29:36,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:36,676 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:36,676 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-26T23:29:36,677 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c707e9ace675,44051,1732663776581 from backup master directory 2024-11-26T23:29:36,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c707e9ace675,44051,1732663776581 2024-11-26T23:29:36,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:29:36,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-26T23:29:36,679 WARN [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:29:36,679 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c707e9ace675,44051,1732663776581 2024-11-26T23:29:36,683 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/hbase.id] with ID: f05af0dd-0de2-49a7-b881-6069a6105646 2024-11-26T23:29:36,683 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/.tmp/hbase.id 2024-11-26T23:29:36,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:29:36,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741826_1002 (size=42) 2024-11-26T23:29:36,691 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/.tmp/hbase.id]:[hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/hbase.id] 2024-11-26T23:29:36,700 INFO [master/c707e9ace675:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:29:36,700 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-26T23:29:36,702 INFO [master/c707e9ace675:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 1ms. 2024-11-26T23:29:36,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:36,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:36,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:29:36,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741827_1003 (size=196) 2024-11-26T23:29:36,710 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-26T23:29:36,711 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-26T23:29:36,711 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:29:36,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:29:36,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741828_1004 (size=1189) 2024-11-26T23:29:36,721 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store 2024-11-26T23:29:36,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:29:36,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741829_1005 (size=34) 2024-11-26T23:29:36,729 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:29:36,729 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:29:36,729 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:36,729 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:36,729 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:29:36,729 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:36,729 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:36,729 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663776729Disabling compacts and flushes for region at 1732663776729Disabling writes for close at 1732663776729Writing region close event to WAL at 1732663776729Closed at 1732663776729 2024-11-26T23:29:36,730 WARN [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/.initializing 2024-11-26T23:29:36,730 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/WALs/c707e9ace675,44051,1732663776581 2024-11-26T23:29:36,732 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C44051%2C1732663776581, suffix=, logDir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/WALs/c707e9ace675,44051,1732663776581, archiveDir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/oldWALs, maxLogs=10 2024-11-26T23:29:36,732 INFO [master/c707e9ace675:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C44051%2C1732663776581.1732663776732 2024-11-26T23:29:36,736 INFO [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/WALs/c707e9ace675,44051,1732663776581/c707e9ace675%2C44051%2C1732663776581.1732663776732 2024-11-26T23:29:36,741 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36725:36725),(127.0.0.1/127.0.0.1:46361:46361)] 2024-11-26T23:29:36,744 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:29:36,744 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:29:36,744 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,744 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,746 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-26T23:29:36,747 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:36,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:29:36,747 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,748 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-26T23:29:36,748 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:36,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:29:36,749 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-26T23:29:36,750 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:36,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:29:36,750 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-26T23:29:36,751 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:36,751 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-26T23:29:36,752 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,752 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,752 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,753 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,753 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,754 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-26T23:29:36,755 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-26T23:29:36,757 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:29:36,757 INFO [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=795737, jitterRate=0.01183195412158966}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-26T23:29:36,758 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732663776744Initializing all the Stores at 1732663776745 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663776745Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663776745Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663776745Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663776745Cleaning up temporary data from old regions at 1732663776753 (+8 ms)Region opened successfully at 1732663776758 (+5 ms) 2024-11-26T23:29:36,758 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-26T23:29:36,761 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@113c83bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:29:36,761 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-26T23:29:36,762 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-26T23:29:36,762 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-26T23:29:36,762 INFO [master/c707e9ace675:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-26T23:29:36,762 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-26T23:29:36,762 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 0 msec 2024-11-26T23:29:36,763 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-26T23:29:36,764 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-26T23:29:36,765 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-26T23:29:36,766 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-26T23:29:36,766 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-26T23:29:36,767 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-26T23:29:36,768 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-26T23:29:36,768 INFO [master/c707e9ace675:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-26T23:29:36,769 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-26T23:29:36,772 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-26T23:29:36,773 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-26T23:29:36,774 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-26T23:29:36,776 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-26T23:29:36,777 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-26T23:29:36,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:29:36,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-26T23:29:36,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:36,778 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:36,779 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c707e9ace675,44051,1732663776581, sessionid=0x101569cffc50000, setting cluster-up flag (Was=false) 2024-11-26T23:29:36,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:36,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:36,788 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-26T23:29:36,789 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,44051,1732663776581 2024-11-26T23:29:36,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:36,792 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:36,797 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-26T23:29:36,798 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c707e9ace675,44051,1732663776581 2024-11-26T23:29:36,799 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-26T23:29:36,800 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-26T23:29:36,800 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-26T23:29:36,800 INFO [master/c707e9ace675:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-26T23:29:36,801 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c707e9ace675,44051,1732663776581 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-26T23:29:36,802 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:29:36,802 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:29:36,802 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:29:36,802 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c707e9ace675:0, corePoolSize=5, maxPoolSize=5 2024-11-26T23:29:36,802 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c707e9ace675:0, corePoolSize=10, maxPoolSize=10 2024-11-26T23:29:36,802 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,802 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:29:36,802 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,803 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732663806803 2024-11-26T23:29:36,803 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-26T23:29:36,803 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-26T23:29:36,803 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-26T23:29:36,803 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-26T23:29:36,803 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-26T23:29:36,803 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-26T23:29:36,803 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,803 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:29:36,804 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-26T23:29:36,804 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-26T23:29:36,804 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-26T23:29:36,804 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-26T23:29:36,804 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-26T23:29:36,804 INFO [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-26T23:29:36,804 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663776804,5,FailOnTimeoutGroup] 2024-11-26T23:29:36,804 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663776804,5,FailOnTimeoutGroup] 2024-11-26T23:29:36,804 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,804 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-26T23:29:36,804 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,804 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,805 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:36,805 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-26T23:29:36,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:29:36,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741831_1007 (size=1321) 2024-11-26T23:29:36,815 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-26T23:29:36,815 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b 2024-11-26T23:29:36,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:29:36,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741832_1008 (size=32) 2024-11-26T23:29:36,822 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:29:36,823 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:29:36,825 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:29:36,825 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:36,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:29:36,825 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:29:36,826 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:29:36,826 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:36,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:29:36,827 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:29:36,828 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:29:36,828 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:36,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:29:36,828 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:29:36,829 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:29:36,829 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:36,830 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:29:36,830 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:29:36,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/1588230740 2024-11-26T23:29:36,831 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/1588230740 2024-11-26T23:29:36,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:29:36,832 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:29:36,832 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:29:36,833 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:29:36,835 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-26T23:29:36,835 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=842141, jitterRate=0.07083871960639954}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:29:36,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732663776822Initializing all the Stores at 1732663776823 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663776823Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663776823Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663776823Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663776823Cleaning up temporary data from old regions at 1732663776832 (+9 ms)Region opened successfully at 1732663776836 (+4 ms) 2024-11-26T23:29:36,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:29:36,836 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:29:36,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:29:36,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:29:36,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:29:36,836 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:29:36,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663776836Disabling compacts and flushes for region at 1732663776836Disabling writes for close at 1732663776836Writing region close event to WAL at 1732663776836Closed at 1732663776836 2024-11-26T23:29:36,837 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:29:36,837 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-26T23:29:36,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-26T23:29:36,838 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:29:36,839 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-26T23:29:36,862 INFO [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(746): ClusterId : f05af0dd-0de2-49a7-b881-6069a6105646 2024-11-26T23:29:36,862 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-26T23:29:36,864 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-26T23:29:36,864 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-26T23:29:36,865 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-26T23:29:36,866 DEBUG [RS:0;c707e9ace675:36141 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a08e457, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c707e9ace675/172.17.0.2:0 2024-11-26T23:29:36,878 DEBUG [RS:0;c707e9ace675:36141 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c707e9ace675:36141 2024-11-26T23:29:36,878 INFO [RS:0;c707e9ace675:36141 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-26T23:29:36,878 INFO [RS:0;c707e9ace675:36141 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-26T23:29:36,878 DEBUG [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-26T23:29:36,878 INFO [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(2659): reportForDuty to master=c707e9ace675,44051,1732663776581 with port=36141, startcode=1732663776636 2024-11-26T23:29:36,878 DEBUG [RS:0;c707e9ace675:36141 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-26T23:29:36,881 INFO [HMaster-EventLoopGroup-16-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39055, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.7 (auth:SIMPLE), service=RegionServerStatusService 2024-11-26T23:29:36,881 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44051 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c707e9ace675,36141,1732663776636 2024-11-26T23:29:36,881 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=44051 {}] master.ServerManager(517): Registering regionserver=c707e9ace675,36141,1732663776636 2024-11-26T23:29:36,883 DEBUG [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b 2024-11-26T23:29:36,883 DEBUG [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46579 2024-11-26T23:29:36,883 DEBUG [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-26T23:29:36,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:29:36,886 DEBUG [RS:0;c707e9ace675:36141 {}] zookeeper.ZKUtil(111): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c707e9ace675,36141,1732663776636 2024-11-26T23:29:36,886 WARN [RS:0;c707e9ace675:36141 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-26T23:29:36,886 INFO [RS:0;c707e9ace675:36141 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:29:36,886 DEBUG [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/WALs/c707e9ace675,36141,1732663776636 2024-11-26T23:29:36,886 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c707e9ace675,36141,1732663776636] 2024-11-26T23:29:36,889 INFO [RS:0;c707e9ace675:36141 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-26T23:29:36,890 INFO [RS:0;c707e9ace675:36141 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-26T23:29:36,890 INFO [RS:0;c707e9ace675:36141 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-26T23:29:36,890 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,891 INFO [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-26T23:29:36,891 INFO [RS:0;c707e9ace675:36141 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-26T23:29:36,891 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,891 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,891 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,891 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,891 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,891 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,891 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c707e9ace675:0, corePoolSize=2, maxPoolSize=2 2024-11-26T23:29:36,891 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,891 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,891 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,891 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,892 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,892 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c707e9ace675:0, corePoolSize=1, maxPoolSize=1 2024-11-26T23:29:36,892 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:29:36,892 DEBUG [RS:0;c707e9ace675:36141 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c707e9ace675:0, corePoolSize=3, maxPoolSize=3 2024-11-26T23:29:36,892 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,892 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,892 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,892 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,892 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,892 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,36141,1732663776636-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:29:36,906 INFO [RS:0;c707e9ace675:36141 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-26T23:29:36,906 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,36141,1732663776636-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,906 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,906 INFO [RS:0;c707e9ace675:36141 {}] regionserver.Replication(171): c707e9ace675,36141,1732663776636 started 2024-11-26T23:29:36,920 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:36,920 INFO [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(1482): Serving as c707e9ace675,36141,1732663776636, RpcServer on c707e9ace675/172.17.0.2:36141, sessionid=0x101569cffc50001 2024-11-26T23:29:36,920 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-26T23:29:36,920 DEBUG [RS:0;c707e9ace675:36141 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c707e9ace675,36141,1732663776636 2024-11-26T23:29:36,920 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,36141,1732663776636' 2024-11-26T23:29:36,920 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-26T23:29:36,920 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-26T23:29:36,921 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-26T23:29:36,921 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-26T23:29:36,921 DEBUG [RS:0;c707e9ace675:36141 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c707e9ace675,36141,1732663776636 2024-11-26T23:29:36,921 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c707e9ace675,36141,1732663776636' 2024-11-26T23:29:36,921 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-26T23:29:36,921 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-26T23:29:36,921 DEBUG [RS:0;c707e9ace675:36141 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-26T23:29:36,921 INFO [RS:0;c707e9ace675:36141 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-26T23:29:36,921 INFO [RS:0;c707e9ace675:36141 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-26T23:29:36,989 WARN [c707e9ace675:44051 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-26T23:29:37,023 INFO [RS:0;c707e9ace675:36141 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C36141%2C1732663776636, suffix=, logDir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/WALs/c707e9ace675,36141,1732663776636, archiveDir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/oldWALs, maxLogs=32 2024-11-26T23:29:37,024 INFO [RS:0;c707e9ace675:36141 {}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C36141%2C1732663776636.1732663777023 2024-11-26T23:29:37,029 INFO [RS:0;c707e9ace675:36141 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/WALs/c707e9ace675,36141,1732663776636/c707e9ace675%2C36141%2C1732663776636.1732663777023 2024-11-26T23:29:37,030 DEBUG [RS:0;c707e9ace675:36141 {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36725:36725),(127.0.0.1/127.0.0.1:46361:46361)] 2024-11-26T23:29:37,240 DEBUG [c707e9ace675:44051 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-26T23:29:37,240 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c707e9ace675,36141,1732663776636 2024-11-26T23:29:37,241 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,36141,1732663776636, state=OPENING 2024-11-26T23:29:37,243 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-26T23:29:37,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:37,244 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:37,245 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-26T23:29:37,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:29:37,245 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:29:37,245 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,36141,1732663776636}] 2024-11-26T23:29:37,245 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,37097,1732663596093/c707e9ace675%2C37097%2C1732663596093.1732663596289 java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:37,245 WARN [Close-WAL-Writer-0 {}] util.RecoverLeaseFSUtils(258): Failed invocation for hdfs://localhost:34607/user/jenkins/test-data/7ab81915-377a-a763-a9ff-ae6111aef4ad/WALs/c707e9ace675,40529,1732663595141/c707e9ace675%2C40529%2C1732663595141.meta.1732663595933.meta java.lang.reflect.InvocationTargetException: null at jdk.internal.reflect.GeneratedMethodAccessor106.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.isFileClosed(RecoverLeaseFSUtils.java:254) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverDFSFileLease(RecoverLeaseFSUtils.java:186) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.RecoverLeaseFSUtils.recoverFileLease(RecoverLeaseFSUtils.java:96) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.recoverLease(AbstractFSWAL.java:2031) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractFSWAL.lambda$closeWriter$17(AbstractFSWAL.java:2044) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.io.IOException: Filesystem closed at org.apache.hadoop.hdfs.DFSClient.checkOpen(DFSClient.java:490) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.isFileClosed(DFSClient.java:1808) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2539) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem$46.doCall(DistributedFileSystem.java:2536) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.isFileClosed(DistributedFileSystem.java:2554) ~[hadoop-hdfs-client-3.4.1.jar:?] ... 11 more 2024-11-26T23:29:37,397 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-26T23:29:37,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43985, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-26T23:29:37,402 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-26T23:29:37,402 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:29:37,404 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c707e9ace675%2C36141%2C1732663776636.meta, suffix=.meta, logDir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/WALs/c707e9ace675,36141,1732663776636, archiveDir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/oldWALs, maxLogs=32 2024-11-26T23:29:37,404 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor c707e9ace675%2C36141%2C1732663776636.meta.1732663777404.meta 2024-11-26T23:29:37,409 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/WALs/c707e9ace675,36141,1732663776636/c707e9ace675%2C36141%2C1732663776636.meta.1732663777404.meta 2024-11-26T23:29:37,409 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36725:36725),(127.0.0.1/127.0.0.1:46361:46361)] 2024-11-26T23:29:37,410 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-26T23:29:37,410 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-26T23:29:37,410 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-26T23:29:37,411 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-26T23:29:37,411 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-26T23:29:37,411 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-26T23:29:37,411 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-26T23:29:37,411 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-26T23:29:37,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-26T23:29:37,412 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-26T23:29:37,412 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:37,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:29:37,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-26T23:29:37,413 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-26T23:29:37,414 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:37,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:29:37,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-26T23:29:37,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-26T23:29:37,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:37,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:29:37,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-26T23:29:37,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-26T23:29:37,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-26T23:29:37,416 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-26T23:29:37,416 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-26T23:29:37,416 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/1588230740 2024-11-26T23:29:37,417 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/1588230740 2024-11-26T23:29:37,418 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-26T23:29:37,418 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-26T23:29:37,419 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-26T23:29:37,420 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-26T23:29:37,420 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=16384, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=738214, jitterRate=-0.06131240725517273}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-26T23:29:37,420 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-26T23:29:37,421 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732663777411Writing region info on filesystem at 1732663777411Initializing all the Stores at 1732663777411Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663777411Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663777412 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732663777412Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732663777412Cleaning up temporary data from old regions at 1732663777418 (+6 ms)Running coprocessor post-open hooks at 1732663777420 (+2 ms)Region opened successfully at 1732663777421 (+1 ms) 2024-11-26T23:29:37,422 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732663777397 2024-11-26T23:29:37,424 DEBUG [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-26T23:29:37,424 INFO [RS_OPEN_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-26T23:29:37,424 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c707e9ace675,36141,1732663776636 2024-11-26T23:29:37,425 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c707e9ace675,36141,1732663776636, state=OPEN 2024-11-26T23:29:37,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:29:37,435 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-26T23:29:37,435 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c707e9ace675,36141,1732663776636 2024-11-26T23:29:37,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:29:37,435 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-26T23:29:37,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-26T23:29:37,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c707e9ace675,36141,1732663776636 in 190 msec 2024-11-26T23:29:37,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-26T23:29:37,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 600 msec 2024-11-26T23:29:37,440 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-26T23:29:37,440 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-26T23:29:37,441 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:29:37,441 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,36141,1732663776636, seqNum=-1] 2024-11-26T23:29:37,441 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:29:37,442 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37265, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:29:37,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 646 msec 2024-11-26T23:29:37,446 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732663777446, completionTime=-1 2024-11-26T23:29:37,446 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-26T23:29:37,446 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-26T23:29:37,448 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-26T23:29:37,448 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732663837448 2024-11-26T23:29:37,448 INFO [master/c707e9ace675:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732663897448 2024-11-26T23:29:37,448 INFO [master/c707e9ace675:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 1 msec 2024-11-26T23:29:37,449 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44051,1732663776581-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:37,449 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44051,1732663776581-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:37,449 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44051,1732663776581-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:37,449 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c707e9ace675:44051, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:37,449 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:37,449 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:37,450 DEBUG [master/c707e9ace675:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-26T23:29:37,452 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 0.773sec 2024-11-26T23:29:37,452 INFO [master/c707e9ace675:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-26T23:29:37,452 INFO [master/c707e9ace675:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-26T23:29:37,452 INFO [master/c707e9ace675:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-26T23:29:37,452 INFO [master/c707e9ace675:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-26T23:29:37,452 INFO [master/c707e9ace675:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-26T23:29:37,452 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44051,1732663776581-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-26T23:29:37,453 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44051,1732663776581-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-26T23:29:37,454 DEBUG [master/c707e9ace675:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-26T23:29:37,454 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-26T23:29:37,454 INFO [master/c707e9ace675:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c707e9ace675,44051,1732663776581-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-26T23:29:37,462 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c9b811e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:29:37,462 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c707e9ace675,44051,-1 for getting cluster id 2024-11-26T23:29:37,462 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-26T23:29:37,464 DEBUG [HMaster-EventLoopGroup-16-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'f05af0dd-0de2-49a7-b881-6069a6105646' 2024-11-26T23:29:37,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-26T23:29:37,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "f05af0dd-0de2-49a7-b881-6069a6105646" 2024-11-26T23:29:37,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5650953b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:29:37,464 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c707e9ace675,44051,-1] 2024-11-26T23:29:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-26T23:29:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:29:37,466 INFO [HMaster-EventLoopGroup-16-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-26T23:29:37,466 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11ec225b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-26T23:29:37,467 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-26T23:29:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c707e9ace675,36141,1732663776636, seqNum=-1] 2024-11-26T23:29:37,468 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-26T23:29:37,468 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-17-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-26T23:29:37,470 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c707e9ace675,44051,1732663776581 2024-11-26T23:29:37,470 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-26T23:29:37,472 INFO [Time-limited test {}] master.MasterRpcServices(567): Client=null/null set balanceSwitch=false 2024-11-26T23:29:37,472 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.FSHLogProvider 2024-11-26T23:29:37,474 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=test.com%2C8080%2C1, suffix=, logDir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/WALs/test.com,8080,1, archiveDir=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/oldWALs, maxLogs=32 2024-11-26T23:29:37,474 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732663777474 2024-11-26T23:29:37,479 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/WALs/test.com,8080,1/test.com%2C8080%2C1.1732663777474 2024-11-26T23:29:37,480 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36725:36725),(127.0.0.1/127.0.0.1:46361:46361)] 2024-11-26T23:29:37,480 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor test.com%2C8080%2C1.1732663777480 2024-11-26T23:29:37,485 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,485 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,485 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,485 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,485 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,485 INFO [Time-limited test {}] wal.AbstractFSWAL(987): Rolled WAL /user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/WALs/test.com,8080,1/test.com%2C8080%2C1.1732663777474 with entries=0, filesize=85 B; new WAL /user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/WALs/test.com,8080,1/test.com%2C8080%2C1.1732663777480 2024-11-26T23:29:37,486 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new FSHLog writer with pipeline: [(127.0.0.1/127.0.0.1:36725:36725),(127.0.0.1/127.0.0.1:46361:46361)] 2024-11-26T23:29:37,486 DEBUG [Time-limited test {}] wal.AbstractFSWAL(879): hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/WALs/test.com,8080,1/test.com%2C8080%2C1.1732663777474 is not closed yet, will try archiving it next time 2024-11-26T23:29:37,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741835_1011 (size=93) 2024-11-26T23:29:37,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741835_1011 (size=93) 2024-11-26T23:29:37,487 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,487 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,487 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,487 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,487 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741836_1012 (size=93) 2024-11-26T23:29:37,489 INFO [WAL-Archive-0 {}] wal.AbstractFSWAL(968): Archiving hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/WALs/test.com,8080,1/test.com%2C8080%2C1.1732663777474 to hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/oldWALs/test.com%2C8080%2C1.1732663777474 2024-11-26T23:29:37,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741836_1012 (size=93) 2024-11-26T23:29:37,492 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/oldWALs 2024-11-26T23:29:37,493 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog test.com%2C8080%2C1:(num 1732663777480) 2024-11-26T23:29:37,493 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-26T23:29:37,493 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:29:37,493 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:29:37,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:29:37,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:29:37,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-26T23:29:37,493 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-26T23:29:37,493 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1936804614, stopped=false 2024-11-26T23:29:37,493 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c707e9ace675,44051,1732663776581 2024-11-26T23:29:37,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:29:37,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-26T23:29:37,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:37,495 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:37,495 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:29:37,495 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-26T23:29:37,495 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestLogRolling.tearDown(AbstractTestLogRolling.java:163) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:29:37,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:29:37,495 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c707e9ace675,36141,1732663776636' ***** 2024-11-26T23:29:37,495 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-26T23:29:37,495 INFO [RS:0;c707e9ace675:36141 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-26T23:29:37,496 INFO [RS:0;c707e9ace675:36141 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-26T23:29:37,496 INFO [RS:0;c707e9ace675:36141 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-26T23:29:37,496 INFO [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(959): stopping server c707e9ace675,36141,1732663776636 2024-11-26T23:29:37,496 INFO [RS:0;c707e9ace675:36141 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:29:37,496 INFO [RS:0;c707e9ace675:36141 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c707e9ace675:36141. 2024-11-26T23:29:37,496 DEBUG [RS:0;c707e9ace675:36141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-26T23:29:37,496 DEBUG [RS:0;c707e9ace675:36141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:29:37,496 INFO [RS:0;c707e9ace675:36141 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-26T23:29:37,496 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:29:37,496 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-26T23:29:37,496 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-26T23:29:37,496 INFO [RS:0;c707e9ace675:36141 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-26T23:29:37,496 INFO [RS:0;c707e9ace675:36141 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-26T23:29:37,496 INFO [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-26T23:29:37,497 INFO [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-26T23:29:37,497 DEBUG [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-26T23:29:37,497 DEBUG [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-26T23:29:37,497 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-26T23:29:37,497 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-26T23:29:37,497 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-26T23:29:37,497 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-26T23:29:37,497 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-26T23:29:37,497 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=74 B heapSize=1.22 KB 2024-11-26T23:29:37,514 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/1588230740/.tmp/ns/512e2e87fcec40528063721e8fd19bd4 is 43, key is default/ns:d/1732663777443/Put/seqid=0 2024-11-26T23:29:37,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741837_1013 (size=5153) 2024-11-26T23:29:37,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741837_1013 (size=5153) 2024-11-26T23:29:37,519 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/1588230740/.tmp/ns/512e2e87fcec40528063721e8fd19bd4 2024-11-26T23:29:37,525 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/1588230740/.tmp/ns/512e2e87fcec40528063721e8fd19bd4 as hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/1588230740/ns/512e2e87fcec40528063721e8fd19bd4 2024-11-26T23:29:37,529 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/1588230740/ns/512e2e87fcec40528063721e8fd19bd4, entries=2, sequenceid=6, filesize=5.0 K 2024-11-26T23:29:37,530 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=6, compaction requested=false 2024-11-26T23:29:37,530 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-26T23:29:37,534 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/data/hbase/meta/1588230740/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-26T23:29:37,534 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-26T23:29:37,534 INFO [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-26T23:29:37,535 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732663777497Running coprocessor pre-close hooks at 1732663777497Disabling compacts and flushes for region at 1732663777497Disabling writes for close at 1732663777497Obtaining lock to block concurrent updates at 1732663777497Preparing flush snapshotting stores in 1588230740 at 1732663777497Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=74, getHeapSize=1184, getOffHeapSize=0, getCellsCount=2 at 1732663777497Flushing stores of hbase:meta,,1.1588230740 at 1732663777498 (+1 ms)Flushing 1588230740/ns: creating writer at 1732663777498Flushing 1588230740/ns: appending metadata at 1732663777514 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1732663777514Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@594c0bd4: reopening flushed file at 1732663777524 (+10 ms)Finished flush of dataSize ~74 B/74, heapSize ~464 B/464, currentSize=0 B/0 for 1588230740 in 33ms, sequenceid=6, compaction requested=false at 1732663777530 (+6 ms)Writing region close event to WAL at 1732663777531 (+1 ms)Running coprocessor post-close hooks at 1732663777534 (+3 ms)Closed at 1732663777534 2024-11-26T23:29:37,535 DEBUG [RS_CLOSE_META-regionserver/c707e9ace675:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-26T23:29:37,697 INFO [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(976): stopping server c707e9ace675,36141,1732663776636; all regions closed. 2024-11-26T23:29:37,697 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,698 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,698 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,698 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,698 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741834_1010 (size=1152) 2024-11-26T23:29:37,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741834_1010 (size=1152) 2024-11-26T23:29:37,703 DEBUG [RS:0;c707e9ace675:36141 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/oldWALs 2024-11-26T23:29:37,703 INFO [RS:0;c707e9ace675:36141 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C36141%2C1732663776636.meta:.meta(num 1732663777404) 2024-11-26T23:29:37,703 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,703 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,703 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,703 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,704 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741833_1009 (size=93) 2024-11-26T23:29:37,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741833_1009 (size=93) 2024-11-26T23:29:37,708 DEBUG [RS:0;c707e9ace675:36141 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/oldWALs 2024-11-26T23:29:37,708 INFO [RS:0;c707e9ace675:36141 {}] wal.AbstractFSWAL(1259): Closed WAL: FSHLog c707e9ace675%2C36141%2C1732663776636:(num 1732663777023) 2024-11-26T23:29:37,708 DEBUG [RS:0;c707e9ace675:36141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-26T23:29:37,708 INFO [RS:0;c707e9ace675:36141 {}] regionserver.LeaseManager(133): Closed leases 2024-11-26T23:29:37,708 INFO [RS:0;c707e9ace675:36141 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:29:37,708 INFO [RS:0;c707e9ace675:36141 {}] hbase.ChoreService(370): Chore service for: regionserver/c707e9ace675:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-26T23:29:37,708 INFO [RS:0;c707e9ace675:36141 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:29:37,708 INFO [regionserver/c707e9ace675:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:29:37,708 INFO [RS:0;c707e9ace675:36141 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36141 2024-11-26T23:29:37,710 INFO [RS:0;c707e9ace675:36141 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:29:37,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-26T23:29:37,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c707e9ace675,36141,1732663776636 2024-11-26T23:29:37,712 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c707e9ace675,36141,1732663776636] 2024-11-26T23:29:37,714 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c707e9ace675,36141,1732663776636 already deleted, retry=false 2024-11-26T23:29:37,714 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c707e9ace675,36141,1732663776636 expired; onlineServers=0 2024-11-26T23:29:37,714 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c707e9ace675,44051,1732663776581' ***** 2024-11-26T23:29:37,714 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-26T23:29:37,714 INFO [M:0;c707e9ace675:44051 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-26T23:29:37,714 INFO [M:0;c707e9ace675:44051 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-26T23:29:37,714 DEBUG [M:0;c707e9ace675:44051 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-26T23:29:37,714 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-26T23:29:37,714 DEBUG [M:0;c707e9ace675:44051 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-26T23:29:37,714 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663776804 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.large.0-1732663776804,5,FailOnTimeoutGroup] 2024-11-26T23:29:37,714 DEBUG [master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663776804 {}] cleaner.HFileCleaner(306): Exit Thread[master/c707e9ace675:0:becomeActiveMaster-HFileCleaner.small.0-1732663776804,5,FailOnTimeoutGroup] 2024-11-26T23:29:37,714 INFO [M:0;c707e9ace675:44051 {}] hbase.ChoreService(370): Chore service for: master/c707e9ace675:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-26T23:29:37,714 INFO [M:0;c707e9ace675:44051 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-26T23:29:37,714 DEBUG [M:0;c707e9ace675:44051 {}] master.HMaster(1795): Stopping service threads 2024-11-26T23:29:37,714 INFO [M:0;c707e9ace675:44051 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-26T23:29:37,714 INFO [M:0;c707e9ace675:44051 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-26T23:29:37,714 INFO [M:0;c707e9ace675:44051 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-26T23:29:37,715 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-26T23:29:37,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-26T23:29:37,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-26T23:29:37,716 DEBUG [M:0;c707e9ace675:44051 {}] zookeeper.ZKUtil(347): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-26T23:29:37,716 WARN [M:0;c707e9ace675:44051 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-26T23:29:37,716 INFO [M:0;c707e9ace675:44051 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/.lastflushedseqids 2024-11-26T23:29:37,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741838_1014 (size=108) 2024-11-26T23:29:37,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741838_1014 (size=108) 2024-11-26T23:29:37,722 INFO [M:0;c707e9ace675:44051 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-26T23:29:37,722 INFO [M:0;c707e9ace675:44051 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-26T23:29:37,722 DEBUG [M:0;c707e9ace675:44051 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-26T23:29:37,722 INFO [M:0;c707e9ace675:44051 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:37,722 DEBUG [M:0;c707e9ace675:44051 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:37,722 DEBUG [M:0;c707e9ace675:44051 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-26T23:29:37,722 DEBUG [M:0;c707e9ace675:44051 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:37,722 INFO [M:0;c707e9ace675:44051 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=7.67 KB heapSize=11.34 KB 2024-11-26T23:29:37,737 DEBUG [M:0;c707e9ace675:44051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/469696edba664f4c8196dbff7b801c92 is 82, key is hbase:meta,,1/info:regioninfo/1732663777424/Put/seqid=0 2024-11-26T23:29:37,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741839_1015 (size=5672) 2024-11-26T23:29:37,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741839_1015 (size=5672) 2024-11-26T23:29:37,742 INFO [M:0;c707e9ace675:44051 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/469696edba664f4c8196dbff7b801c92 2024-11-26T23:29:37,760 DEBUG [M:0;c707e9ace675:44051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5849a07648344e20a9299d91494fa9a8 is 240, key is \x00\x00\x00\x00\x00\x00\x00\x01/proc:d/1732663777446/Put/seqid=0 2024-11-26T23:29:37,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741840_1016 (size=5275) 2024-11-26T23:29:37,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741840_1016 (size=5275) 2024-11-26T23:29:37,765 INFO [M:0;c707e9ace675:44051 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=7.06 KB at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5849a07648344e20a9299d91494fa9a8 2024-11-26T23:29:37,783 DEBUG [M:0;c707e9ace675:44051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24d718fa78b84aecacdd62d1e30ae387 is 69, key is c707e9ace675,36141,1732663776636/rs:state/1732663776881/Put/seqid=0 2024-11-26T23:29:37,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741841_1017 (size=5156) 2024-11-26T23:29:37,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741841_1017 (size=5156) 2024-11-26T23:29:37,788 INFO [M:0;c707e9ace675:44051 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24d718fa78b84aecacdd62d1e30ae387 2024-11-26T23:29:37,805 DEBUG [M:0;c707e9ace675:44051 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3d2114cbfeee481e99e525c7b57155b7 is 52, key is load_balancer_on/state:d/1732663777471/Put/seqid=0 2024-11-26T23:29:37,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741842_1018 (size=5056) 2024-11-26T23:29:37,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741842_1018 (size=5056) 2024-11-26T23:29:37,809 INFO [M:0;c707e9ace675:44051 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=29 (bloomFilter=true), to=hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3d2114cbfeee481e99e525c7b57155b7 2024-11-26T23:29:37,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:29:37,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36141-0x101569cffc50001, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:29:37,812 INFO [RS:0;c707e9ace675:36141 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:29:37,812 INFO [RS:0;c707e9ace675:36141 {}] regionserver.HRegionServer(1031): Exiting; stopping=c707e9ace675,36141,1732663776636; zookeeper connection closed. 2024-11-26T23:29:37,813 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@16cb88ac {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@16cb88ac 2024-11-26T23:29:37,813 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-26T23:29:37,814 DEBUG [M:0;c707e9ace675:44051 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/469696edba664f4c8196dbff7b801c92 as hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/469696edba664f4c8196dbff7b801c92 2024-11-26T23:29:37,817 INFO [M:0;c707e9ace675:44051 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/469696edba664f4c8196dbff7b801c92, entries=8, sequenceid=29, filesize=5.5 K 2024-11-26T23:29:37,818 DEBUG [M:0;c707e9ace675:44051 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/5849a07648344e20a9299d91494fa9a8 as hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5849a07648344e20a9299d91494fa9a8 2024-11-26T23:29:37,822 INFO [M:0;c707e9ace675:44051 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/5849a07648344e20a9299d91494fa9a8, entries=3, sequenceid=29, filesize=5.2 K 2024-11-26T23:29:37,822 DEBUG [M:0;c707e9ace675:44051 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/24d718fa78b84aecacdd62d1e30ae387 as hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/24d718fa78b84aecacdd62d1e30ae387 2024-11-26T23:29:37,825 INFO [M:0;c707e9ace675:44051 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/24d718fa78b84aecacdd62d1e30ae387, entries=1, sequenceid=29, filesize=5.0 K 2024-11-26T23:29:37,826 DEBUG [M:0;c707e9ace675:44051 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/3d2114cbfeee481e99e525c7b57155b7 as hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3d2114cbfeee481e99e525c7b57155b7 2024-11-26T23:29:37,830 INFO [M:0;c707e9ace675:44051 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46579/user/jenkins/test-data/2bf07649-e8d8-e310-8577-2a99e0eb2e2b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/3d2114cbfeee481e99e525c7b57155b7, entries=1, sequenceid=29, filesize=4.9 K 2024-11-26T23:29:37,831 INFO [M:0;c707e9ace675:44051 {}] regionserver.HRegion(3140): Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=29, compaction requested=false 2024-11-26T23:29:37,832 INFO [M:0;c707e9ace675:44051 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-26T23:29:37,832 DEBUG [M:0;c707e9ace675:44051 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732663777722Disabling compacts and flushes for region at 1732663777722Disabling writes for close at 1732663777722Obtaining lock to block concurrent updates at 1732663777722Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732663777722Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=7850, getHeapSize=11544, getOffHeapSize=0, getCellsCount=36 at 1732663777722Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732663777723 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732663777723Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732663777737 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732663777737Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732663777746 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732663777760 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732663777760Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732663777769 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732663777782 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732663777782Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732663777791 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732663777804 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732663777805 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@641cdc56: reopening flushed file at 1732663777813 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23175b05: reopening flushed file at 1732663777818 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f42f23b: reopening flushed file at 1732663777822 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d22dd5f: reopening flushed file at 1732663777826 (+4 ms)Finished flush of dataSize ~7.67 KB/7850, heapSize ~11.27 KB/11544, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 109ms, sequenceid=29, compaction requested=false at 1732663777831 (+5 ms)Writing region close event to WAL at 1732663777832 (+1 ms)Closed at 1732663777832 2024-11-26T23:29:37,834 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,835 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,835 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,835 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,835 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-26T23:29:37,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39645 is added to blk_1073741830_1006 (size=10311) 2024-11-26T23:29:37,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43415 is added to blk_1073741830_1006 (size=10311) 2024-11-26T23:29:37,837 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-26T23:29:37,837 INFO [M:0;c707e9ace675:44051 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-26T23:29:37,837 INFO [M:0;c707e9ace675:44051 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44051 2024-11-26T23:29:37,837 INFO [M:0;c707e9ace675:44051 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-26T23:29:37,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:29:37,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:44051-0x101569cffc50000, quorum=127.0.0.1:60826, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-26T23:29:37,939 INFO [M:0;c707e9ace675:44051 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-26T23:29:37,942 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@663b7fb1{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:29:37,942 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@644054b{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:29:37,942 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:29:37,943 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@47b9368{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:29:37,943 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@e9b8f9f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/hadoop.log.dir/,STOPPED} 2024-11-26T23:29:37,944 WARN [BP-364958726-172.17.0.2-1732663775802 heartbeating to localhost/127.0.0.1:46579 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:29:37,944 WARN [BP-364958726-172.17.0.2-1732663775802 heartbeating to localhost/127.0.0.1:46579 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-364958726-172.17.0.2-1732663775802 (Datanode Uuid f6b642ef-988d-4f85-b9f3-d1cfe9844422) service to localhost/127.0.0.1:46579 2024-11-26T23:29:37,944 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:29:37,944 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:29:37,945 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd/data/data3/current/BP-364958726-172.17.0.2-1732663775802 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:29:37,945 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd/data/data4/current/BP-364958726-172.17.0.2-1732663775802 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:29:37,945 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:29:37,947 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2482618b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-26T23:29:37,948 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@56836339{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:29:37,948 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:29:37,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6162294b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:29:37,948 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40f3733a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/hadoop.log.dir/,STOPPED} 2024-11-26T23:29:37,949 WARN [BP-364958726-172.17.0.2-1732663775802 heartbeating to localhost/127.0.0.1:46579 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-26T23:29:37,949 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-26T23:29:37,949 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-26T23:29:37,949 WARN [BP-364958726-172.17.0.2-1732663775802 heartbeating to localhost/127.0.0.1:46579 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-364958726-172.17.0.2-1732663775802 (Datanode Uuid bcbd0e77-2b0d-45c8-a337-86b48680f756) service to localhost/127.0.0.1:46579 2024-11-26T23:29:37,950 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd/data/data1/current/BP-364958726-172.17.0.2-1732663775802 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:29:37,950 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/cluster_c1f7c300-a58f-9c98-4979-51b551f868bd/data/data2/current/BP-364958726-172.17.0.2-1732663775802 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-26T23:29:37,950 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-26T23:29:37,956 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3885c0c5{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-26T23:29:37,956 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2394ff19{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-26T23:29:37,956 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-26T23:29:37,957 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5031c9de{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-26T23:29:37,957 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4d118eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/72829276-1369-a082-2e38-0f6d90cee9f8/hadoop.log.dir/,STOPPED} 2024-11-26T23:29:37,963 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-26T23:29:37,976 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-26T23:29:37,986 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestLogRolling#testLogRollOnNothingWritten Thread=270 (was 232) Potentially hanging thread: nioEventLoopGroup-43-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46579 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-2 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.hfs.7@localhost:46579 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-42-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-43-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46579 from jenkins.hfs.7 java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-42-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: globalEventExecutor-1-20 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) app//io.netty.util.concurrent.GlobalEventExecutor.takeTask(GlobalEventExecutor.java:113) app//io.netty.util.concurrent.GlobalEventExecutor$TaskRunner.run(GlobalEventExecutor.java:259) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46579 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46579 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-44-1 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (75657370) connection to localhost/127.0.0.1:46579 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: nioEventLoopGroup-43-2 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: MiniHBaseClusterRegionServer-EventLoopGroup-17-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-1 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: nioEventLoopGroup-45-3 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//io.netty.util.concurrent.SingleThreadEventExecutor.confirmShutdown(SingleThreadEventExecutor.java:787) app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:596) app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins@localhost:46579 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46579 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HMaster-EventLoopGroup-16-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=537 (was 512) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=79 (was 79), ProcessCount=11 (was 11), AvailableMemoryMB=6394 (was 6421)